From f1bcfc7868db021ad1780bed41b4f04271e1b2c4 Mon Sep 17 00:00:00 2001 From: better629 Date: Fri, 27 Oct 2023 17:06:28 +0800 Subject: [PATCH 001/135] add docs/tutorial --- docs/tutorial/faq.md | 6 +++ docs/tutorial/usage.md | 67 +++++++++++++++++++++++++++++++ docs/tutorial/usage_cn.md | 63 +++++++++++++++++++++++++++++ docs/tutorial/what_can_this_do.md | 0 4 files changed, 136 insertions(+) create mode 100644 docs/tutorial/faq.md create mode 100644 docs/tutorial/usage.md create mode 100644 docs/tutorial/usage_cn.md create mode 100644 docs/tutorial/what_can_this_do.md diff --git a/docs/tutorial/faq.md b/docs/tutorial/faq.md new file mode 100644 index 000000000..0d2fa7ef0 --- /dev/null +++ b/docs/tutorial/faq.md @@ -0,0 +1,6 @@ +## FAQs + +### installation + + +### openai usage diff --git a/docs/tutorial/usage.md b/docs/tutorial/usage.md new file mode 100644 index 000000000..ee87b65c9 --- /dev/null +++ b/docs/tutorial/usage.md @@ -0,0 +1,67 @@ +## MetaGPT Usage + +### Configuration + +- Configure your `OPENAI_API_KEY` in any of `config/key.yaml / config/config.yaml / env` +- Priority order: `config/key.yaml > config/config.yaml > env` + +```bash +# Copy the configuration file and make the necessary modifications. +cp config/config.yaml config/key.yaml +``` + +| Variable Name | config/key.yaml | env | +| ------------------------------------------ | ----------------------------------------- | ----------------------------------------------- | +| OPENAI_API_KEY # Replace with your own key | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | +| OPENAI_API_BASE # Optional | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | + +### Initiating a startup + +```shell +# Run the script +python startup.py "Write a cli snake game" +# Do not hire an engineer to implement the project +python startup.py "Write a cli snake game" --implement False +# Hire an engineer and perform code reviews +python startup.py "Write a cli snake game" --code_review True +``` + +After running the script, you can find your new project in the `workspace/` directory. + +### Preference of Platform or Tool + +You can tell which platform or tool you want to use when stating your requirements. + +```shell +python startup.py "Write a cli snake game based on pygame" +``` + +### Usage + +``` +NAME + startup.py - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + +SYNOPSIS + startup.py IDEA + +DESCRIPTION + We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + +POSITIONAL ARGUMENTS + IDEA + Type: str + Your innovative idea, such as "Creating a snake game." + +FLAGS + --investment=INVESTMENT + Type: float + Default: 3.0 + As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. + --n_round=N_ROUND + Type: int + Default: 5 + +NOTES + You can also use flags syntax for POSITIONAL ARGUMENTS +``` \ No newline at end of file diff --git a/docs/tutorial/usage_cn.md b/docs/tutorial/usage_cn.md new file mode 100644 index 000000000..4b3bdd2c3 --- /dev/null +++ b/docs/tutorial/usage_cn.md @@ -0,0 +1,63 @@ +## MetaGPT 使用 + +### 配置 + +- 在 `config/key.yaml / config/config.yaml / env` 中配置您的 `OPENAI_API_KEY` +- 优先级顺序：`config/key.yaml > config/config.yaml > env` + +```bash +# 复制配置文件并进行必要的修改 +cp config/config.yaml config/key.yaml +``` + +| 变量名 | config/key.yaml | env | +| ----------------------------------- | ----------------------------------------- | ----------------------------------------------- | +| OPENAI_API_KEY # 用您自己的密钥替换 | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | +| OPENAI_API_BASE # 可选 | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | + +### 示例：启动一个创业公司 + +```shell +python startup.py "写一个命令行贪吃蛇" +# 开启code review模式会花费更多的金钱, 但是会提升代码质量和成功率 +python startup.py "写一个命令行贪吃蛇" --code_review True +``` + +运行脚本后，您可以在 `workspace/` 目录中找到您的新项目。 + +### 平台或工具的倾向性 +可以在阐述需求时说明想要使用的平台或工具。 +例如： +```shell +python startup.py "写一个基于pygame的命令行贪吃蛇" +``` + +### 使用 + +``` +名称 + startup.py - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 + +概要 + startup.py IDEA + +描述 + 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 + +位置参数 + IDEA + 类型: str + 您的创新想法，例如"写一个命令行贪吃蛇。" + +标志 + --investment=INVESTMENT + 类型: float + 默认值: 3.0 + 作为投资者，您有机会向这家AI公司投入一定的美元金额。 + --n_round=N_ROUND + 类型: int + 默认值: 5 + +备注 + 您也可以用`标志`的语法，来处理`位置参数` +``` diff --git a/docs/tutorial/what_can_this_do.md b/docs/tutorial/what_can_this_do.md new file mode 100644 index 000000000..e69de29bb From 11e720a496cf3d63d19bc6f9a606113b02ecd7cc Mon Sep 17 00:00:00 2001 From: better629 Date: Fri, 27 Oct 2023 17:06:45 +0800 Subject: [PATCH 002/135] add docs/install --- docs/install/cli_install.md | 109 ++++++++++++++++++++++++++++++ docs/install/cli_install_cn.md | 43 ++++++++++++ docs/install/docker_install.md | 44 ++++++++++++ docs/install/docker_install_cn.md | 44 ++++++++++++ 4 files changed, 240 insertions(+) create mode 100644 docs/install/cli_install.md create mode 100644 docs/install/cli_install_cn.md create mode 100644 docs/install/docker_install.md create mode 100644 docs/install/docker_install_cn.md diff --git a/docs/install/cli_install.md b/docs/install/cli_install.md new file mode 100644 index 000000000..19f0e3c75 --- /dev/null +++ b/docs/install/cli_install.md @@ -0,0 +1,109 @@ +## Traditional Command Line Installation + +### Support System and version +| System Version | Python Version | Supported | +| ---- | ---- | ----- | +| macOS 13.x | python 3.9 | Yes | +| Windows 11 | python 3.9 | Yes | +| Ubuntu 22.04 | python 3.9 | Yes | + +### Detail Installation +```bash +# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) +npm --version +sudo npm install -g @mermaid-js/mermaid-cli + +# Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using: +python3 --version + +# Step 3: Clone the repository to your local machine, and install it. +git clone https://github.com/geekan/metagpt +cd metagpt +pip install -e. +``` + +**Note:** + +- If already have Chrome, Chromium, or MS Edge installed, you can skip downloading Chromium by setting the environment variable + `PUPPETEER_SKIP_CHROMIUM_DOWNLOAD` to `true`. + +- Some people are [having issues](https://github.com/mermaidjs/mermaid.cli/issues/15) installing this tool globally. Installing it locally is an alternative solution, + + ```bash + npm install @mermaid-js/mermaid-cli + ``` + +- don't forget to the configuration for mmdc in config.yml + + ```yml + PUPPETEER_CONFIG: "./config/puppeteer-config.json" + MMDC: "./node_modules/.bin/mmdc" + ``` + +- if `pip install -e.` fails with error `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`, try instead running `pip install -e. --user` + +- To convert Mermaid charts to SVG, PNG, and PDF formats. In addition to the Node.js version of Mermaid-CLI, you now have the option to use Python version Playwright, pyppeteer or mermaid.ink for this task. + + - Playwright + - **Install Playwright** + + ```bash + pip install playwright + ``` + + - **Install the Required Browsers** + + to support PDF conversion, please install Chrominum. + + ```bash + playwright install --with-deps chromium + ``` + + - **modify `config.yaml`** + + uncomment MERMAID_ENGINE from config.yaml and change it to `playwright` + + ```yaml + MERMAID_ENGINE: playwright + ``` + + - pyppeteer + - **Install pyppeteer** + + ```bash + pip install pyppeteer + ``` + + - **Use your own Browsers** + + pyppeteer allows you use installed browsers, please set the following envirment + + ```bash + export PUPPETEER_EXECUTABLE_PATH = /path/to/your/chromium or edge or chrome + ``` + + please do not use this command to install browser, it is too old + + ```bash + pyppeteer-install + ``` + + - **modify `config.yaml`** + + uncomment MERMAID_ENGINE from config.yaml and change it to `pyppeteer` + + ```yaml + MERMAID_ENGINE: pyppeteer + ``` + + - mermaid.ink + - **modify `config.yaml`** + + uncomment MERMAID_ENGINE from config.yaml and change it to `ink` + + ```yaml + MERMAID_ENGINE: ink + ``` + + Note: this method does not support pdf export. + \ No newline at end of file diff --git a/docs/install/cli_install_cn.md b/docs/install/cli_install_cn.md new file mode 100644 index 000000000..fe97a0b80 --- /dev/null +++ b/docs/install/cli_install_cn.md @@ -0,0 +1,43 @@ +## 命令行安装 + +### 支持的系统和版本 +| 系统版本 | Python 版本 | 是否支持 | +| ---- | ---- | ----- | +| macOS 13.x | python 3.9 | 是 | +| Windows 11 | python 3.9 | 是 | +| Ubuntu 22.04 | python 3.9 | 是 | + +### 详细安装 + +```bash +# 第 1 步：确保您的系统上安装了 NPM。并使用npm安装mermaid-js +npm --version +sudo npm install -g @mermaid-js/mermaid-cli + +# 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： +python --version + +# 第 3 步：克隆仓库到您的本地机器，并进行安装。 +git clone https://github.com/geekan/metagpt +cd metagpt +pip install -e. +``` + +**注意：** + +- 如果已经安装了Chrome、Chromium或MS Edge，可以通过将环境变量`PUPPETEER_SKIP_CHROMIUM_DOWNLOAD`设置为`true`来跳过下载Chromium。 + +- 一些人在全局安装此工具时遇到问题。在本地安装是替代解决方案， + + ```bash + npm install @mermaid-js/mermaid-cli + ``` + +- 不要忘记在config.yml中为mmdc配置配置， + + ```yml + PUPPETEER_CONFIG: "./config/puppeteer-config.json" + MMDC: "./node_modules/.bin/mmdc" + ``` + +- 如果`pip install -e.`失败并显示错误`[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`，请尝试使用`pip install -e. --user`运行。 diff --git a/docs/install/docker_install.md b/docs/install/docker_install.md new file mode 100644 index 000000000..b803a5dae --- /dev/null +++ b/docs/install/docker_install.md @@ -0,0 +1,44 @@ +## Docker Installation + +### Use default MetaGPT image + +```bash +# Step 1: Download metagpt official image and prepare config.yaml +docker pull metagpt/metagpt:latest +mkdir -p /opt/metagpt/{config,workspace} +docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml +vim /opt/metagpt/config/key.yaml # Change the config + +# Step 2: Run metagpt demo with container +docker run --rm \ + --privileged \ + -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ + -v /opt/metagpt/workspace:/app/metagpt/workspace \ + metagpt/metagpt:latest \ + python3 startup.py "Write a cli snake game" + +# You can also start a container and execute commands in it +docker run --name metagpt -d \ + --privileged \ + -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ + -v /opt/metagpt/workspace:/app/metagpt/workspace \ + metagpt/metagpt:latest + +docker exec -it metagpt /bin/bash +$ python3 startup.py "Write a cli snake game" +``` + +The command `docker run ...` do the following things: + +- Run in privileged mode to have permission to run the browser +- Map host configure file `/opt/metagpt/config/key.yaml` to container `/app/metagpt/config/key.yaml` +- Map host directory `/opt/metagpt/workspace` to container `/app/metagpt/workspace` +- Execute the demo command `python3 startup.py "Write a cli snake game"` + +### Build image by yourself + +```bash +# You can also build metagpt image by yourself. +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT && docker build -t metagpt:custom . +``` diff --git a/docs/install/docker_install_cn.md b/docs/install/docker_install_cn.md new file mode 100644 index 000000000..347fae10c --- /dev/null +++ b/docs/install/docker_install_cn.md @@ -0,0 +1,44 @@ +## Docker安装 + +### 使用MetaGPT镜像 + +```bash +# 步骤1: 下载metagpt官方镜像并准备好config.yaml +docker pull metagpt/metagpt:latest +mkdir -p /opt/metagpt/{config,workspace} +docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml +vim /opt/metagpt/config/key.yaml # 修改配置文件 + +# 步骤2: 使用容器运行metagpt演示 +docker run --rm \ + --privileged \ + -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ + -v /opt/metagpt/workspace:/app/metagpt/workspace \ + metagpt/metagpt:latest \ + python startup.py "Write a cli snake game" + +# 您也可以启动一个容器并在其中执行命令 +docker run --name metagpt -d \ + --privileged \ + -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ + -v /opt/metagpt/workspace:/app/metagpt/workspace \ + metagpt/metagpt:latest + +docker exec -it metagpt /bin/bash +$ python startup.py "Write a cli snake game" +``` + +`docker run ...`做了以下事情: + +- 以特权模式运行，有权限运行浏览器 +- 将主机文件 `/opt/metagpt/config/key.yaml` 映射到容器文件 `/app/metagpt/config/key.yaml` +- 将主机目录 `/opt/metagpt/workspace` 映射到容器目录 `/app/metagpt/workspace` +- 执行示例命令 `python startup.py "Write a cli snake game"` + +### 自己构建镜像 + +```bash +# 您也可以自己构建metagpt镜像 +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT && docker build -t metagpt:custom . +``` From d62cf5cf1d68881d10bb11b5adf2f594a3d3589f Mon Sep 17 00:00:00 2001 From: better629 Date: Fri, 27 Oct 2023 17:08:25 +0800 Subject: [PATCH 003/135] update EN&CN README --- README.md | 280 ++++++---------------------------------- docs/README_CN.md | 194 +++++++--------------------- docs/examples/README.md | 0 3 files changed, 88 insertions(+), 386 deletions(-) create mode 100644 docs/examples/README.md diff --git a/README.md b/README.md index 70460ceb4..f3fb3dff9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ # MetaGPT: The Multi-Agent Framework CN doc EN doc JA doc -Discord Follow +Discord Follow License: MIT roadmap Twitter Follow @@ -33,31 +33,9 @@ # MetaGPT: The Multi-Agent Framework

Software Company Multi-Role Schematic (Gradually Implementing)

-## MetaGPT's Abilities +## Install - -https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 - - - -## Examples (fully generated by GPT-4) - -For example, if you type `python startup.py "Design a RecSys like Toutiao"`, you would get many outputs, one of them is data & api design - -![Jinri Toutiao Recsys Data & API Design](docs/resources/workspace/content_rec_sys/resources/data_api_design.png) - -It costs approximately **$0.2** (in GPT-4 API fees) to generate one example with analysis and design, and around **$2.0** for a full project. - - - - -## Installation - -### Installation Video Guide - -- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) - -### Traditional Installation +### Conda installation ```bash # Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) @@ -65,100 +43,24 @@ # Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. sudo npm install -g @mermaid-js/mermaid-cli # Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using: -python --version +# You can use conda to initialize a new python env +# conda create -n metagpt python=3.9 +# conda activate metagpt +python3 --version # Step 3: Clone the repository to your local machine, and install it. git clone https://github.com/geekan/metagpt cd metagpt -pip install -e. +pip3 install -e. + +# Step 4: run the startup.py +# setup your OPENAI_API_KEY in key.yaml copy from config.yaml +python3 startup.py "Write a cli snake game" ``` -**Note:** +detail installation please refer to [cli_install](docs/install/cli_install.md) -- If already have Chrome, Chromium, or MS Edge installed, you can skip downloading Chromium by setting the environment variable - `PUPPETEER_SKIP_CHROMIUM_DOWNLOAD` to `true`. - -- Some people are [having issues](https://github.com/mermaidjs/mermaid.cli/issues/15) installing this tool globally. Installing it locally is an alternative solution, - - ```bash - npm install @mermaid-js/mermaid-cli - ``` - -- don't forget to the configuration for mmdc in config.yml - - ```yml - PUPPETEER_CONFIG: "./config/puppeteer-config.json" - MMDC: "./node_modules/.bin/mmdc" - ``` - -- if `pip install -e.` fails with error `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`, try instead running `pip install -e. --user` - -- To convert Mermaid charts to SVG, PNG, and PDF formats. In addition to the Node.js version of Mermaid-CLI, you now have the option to use Python version Playwright, pyppeteer or mermaid.ink for this task. - - - Playwright - - **Install Playwright** - - ```bash - pip install playwright - ``` - - - **Install the Required Browsers** - - to support PDF conversion, please install Chrominum. - - ```bash - playwright install --with-deps chromium - ``` - - - **modify `config.yaml`** - - uncomment MERMAID_ENGINE from config.yaml and change it to `playwright` - - ```yaml - MERMAID_ENGINE: playwright - ``` - - - pyppeteer - - **Install pyppeteer** - - ```bash - pip install pyppeteer - ``` - - - **Use your own Browsers** - - pyppeteer allows you use installed browsers, please set the following envirment - - ```bash - export PUPPETEER_EXECUTABLE_PATH = /path/to/your/chromium or edge or chrome - ``` - - please do not use this command to install browser, it is too old - - ```bash - pyppeteer-install - ``` - - - **modify `config.yaml`** - - uncomment MERMAID_ENGINE from config.yaml and change it to `pyppeteer` - - ```yaml - MERMAID_ENGINE: pyppeteer - ``` - - - mermaid.ink - - **modify `config.yaml`** - - uncomment MERMAID_ENGINE from config.yaml and change it to `ink` - - ```yaml - MERMAID_ENGINE: ink - ``` - - Note: this method does not support pdf export. - -### Installation by Docker +### Docker installation ```bash # Step 1: Download metagpt official image and prepare config.yaml @@ -174,124 +76,46 @@ # Step 2: Run metagpt demo with container -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ python startup.py "Write a cli snake game" - -# You can also start a container and execute commands in it -docker run --name metagpt -d \ - --privileged \ - -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ - -v /opt/metagpt/workspace:/app/metagpt/workspace \ - metagpt/metagpt:latest - -docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" ``` -The command `docker run ...` do the following things: +detail installation please refer to [docker_install](docs/install/docker_install.md) -- Run in privileged mode to have permission to run the browser -- Map host configure file `/opt/metagpt/config/key.yaml` to container `/app/metagpt/config/key.yaml` -- Map host directory `/opt/metagpt/workspace` to container `/app/metagpt/workspace` -- Execute the demo command `python startup.py "Write a cli snake game"` +### QuickStart & Demo Video +- Try it on [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) +- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) +- [Official Demo Video](https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d) -### Build image by yourself +## Tutorial -```bash -# You can also build metagpt image by yourself. -git clone https://github.com/geekan/MetaGPT.git -cd MetaGPT && docker build -t metagpt:custom . -``` +- [Oneline Document]() +- [Usage](docs/tutorial/usage.md) +- [What can MetaGPT do?](docs/tutorial/what_can_this_do.md) +- How to build your own agents? + - [MetaGPT Usage & Development Guide](https://deepwisdom.feishu.cn/wiki/RUnswqUIPimRJmkkDZ7cLYwOndg#Yu2AdUvymoo67Jxbp0bcu8G4nEb) +- [Contribution](docs/develop/contribution.md) + - Develop RFC + - [Develop Roadmap](docs/ROADMAP.md) +- [Examples](docs/examples/README.md) + - Researcher + - Werewolf Game +- [FAQs](docs/tutorial/faq.md) +- [The generated projects display wall](https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419) -## Configuration +## Support -- Configure your `OPENAI_API_KEY` in any of `config/key.yaml / config/config.yaml / env` -- Priority order: `config/key.yaml > config/config.yaml > env` +### Discard Join US +📢 Join Our [Discord Channel](https://discord.gg/ZRHeExS6xv)! -```bash -# Copy the configuration file and make the necessary modifications. -cp config/config.yaml config/key.yaml -``` +Looking forward to seeing you there! 🎉 -| Variable Name | config/key.yaml | env | -| ------------------------------------------ | ----------------------------------------- | ----------------------------------------------- | -| OPENAI_API_KEY # Replace with your own key | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | -| OPENAI_API_BASE # Optional | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | +### Contact Information -## Tutorial: Initiating a startup +If you have any questions or feedback about this project, please feel free to contact us. We highly appreciate your suggestions! -```shell -# Run the script -python startup.py "Write a cli snake game" -# Do not hire an engineer to implement the project -python startup.py "Write a cli snake game" --implement False -# Hire an engineer and perform code reviews -python startup.py "Write a cli snake game" --code_review True -``` +- **Email:** alexanderwu@fuzhi.ai +- **GitHub Issues:** For more technical inquiries, you can also create a new issue in our [GitHub repository](https://github.com/geekan/metagpt/issues). -After running the script, you can find your new project in the `workspace/` directory. - -### Preference of Platform or Tool - -You can tell which platform or tool you want to use when stating your requirements. - -```shell -python startup.py "Write a cli snake game based on pygame" -``` - -### Usage - -``` -NAME - startup.py - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - -SYNOPSIS - startup.py IDEA - -DESCRIPTION - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - -POSITIONAL ARGUMENTS - IDEA - Type: str - Your innovative idea, such as "Creating a snake game." - -FLAGS - --investment=INVESTMENT - Type: float - Default: 3.0 - As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. - --n_round=N_ROUND - Type: int - Default: 5 - -NOTES - You can also use flags syntax for POSITIONAL ARGUMENTS -``` - -### Code walkthrough - -```python -from metagpt.software_company import SoftwareCompany -from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer - -async def startup(idea: str, investment: float = 3.0, n_round: int = 5): - """Run a startup. Be a boss.""" - company = SoftwareCompany() - company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) -``` - -You can check `examples` for more details on single role (with knowledge base) and LLM only examples. - -## QuickStart - -It is difficult to install and configure the local environment for some users. The following tutorials will allow you to quickly experience the charm of MetaGPT. - -- [MetaGPT quickstart](https://deepwisdom.feishu.cn/wiki/CyY9wdJc4iNqArku3Lncl4v8n2b) - -Try it on Huggingface Space -- https://huggingface.co/spaces/deepwisdom/MetaGPT +We will respond to all questions within 2-3 business days. ## Citation @@ -307,23 +131,3 @@ ## Citation primaryClass={cs.AI} } ``` - -## Contact Information - -If you have any questions or feedback about this project, please feel free to contact us. We highly appreciate your suggestions! - -- **Email:** alexanderwu@fuzhi.ai -- **GitHub Issues:** For more technical inquiries, you can also create a new issue in our [GitHub repository](https://github.com/geekan/metagpt/issues). - -We will respond to all questions within 2-3 business days. - -## Demo - -https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d - -## Join us - -📢 Join Our Discord Channel! -https://discord.gg/ZRHeExS6xv - -Looking forward to seeing you there! 🎉 diff --git a/docs/README_CN.md b/docs/README_CN.md index 9d6f34c11..8c04e5066 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -12,7 +12,7 @@ # MetaGPT: 多智能体框架 CN doc EN doc JA doc -Discord Follow +Discord Follow License: MIT roadmap Twitter Follow @@ -33,22 +33,8 @@ # MetaGPT: 多智能体框架

软件公司多角色示意图（正在逐步实现）

-## MetaGPT 的能力 - -https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 - - -## 示例（均由 GPT-4 生成） - -例如，键入`python startup.py "写个类似今日头条的推荐系统"`并回车，你会获得一系列输出，其一是数据结构与API设计 - -![今日头条 Recsys 数据 & API 设计](resources/workspace/content_rec_sys/resources/data_api_design.png) - -这需要大约**0.2美元**（GPT-4 API的费用）来生成一个带有分析和设计的示例，大约2.0美元用于一个完整的项目 - ## 安装 - -### 传统安装 +### Conda安装 ```bash # 第 1 步：确保您的系统上安装了 NPM。并使用npm安装mermaid-js @@ -56,32 +42,22 @@ # 第 1 步：确保您的系统上安装了 NPM。并使用npm安装mermaid-js sudo npm install -g @mermaid-js/mermaid-cli # 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： +# 可以使用conda来初始化新的python环境 +# conda create -n metagpt python=3.9 +# conda activate metagpt python --version # 第 3 步：克隆仓库到您的本地机器，并进行安装。 git clone https://github.com/geekan/metagpt cd metagpt pip install -e. + +# 第 4 步：执行startup.py +# 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY +python3 startup.py "Write a cli snake game" ``` -**注意：** - -- 如果已经安装了Chrome、Chromium或MS Edge，可以通过将环境变量`PUPPETEER_SKIP_CHROMIUM_DOWNLOAD`设置为`true`来跳过下载Chromium。 - -- 一些人在全局安装此工具时遇到问题。在本地安装是替代解决方案， - - ```bash - npm install @mermaid-js/mermaid-cli - ``` - -- 不要忘记在config.yml中为mmdc配置配置， - - ```yml - PUPPETEER_CONFIG: "./config/puppeteer-config.json" - MMDC: "./node_modules/.bin/mmdc" - ``` - -- 如果`pip install -e.`失败并显示错误`[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`，请尝试使用`pip install -e. --user`运行。 +详细的安装请安装 [cli_install](docs/install/cli_install_cn.md) ### Docker安装 @@ -99,121 +75,39 @@ # 步骤2: 使用容器运行metagpt演示 -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ python startup.py "Write a cli snake game" - -# 您也可以启动一个容器并在其中执行命令 -docker run --name metagpt -d \ - --privileged \ - -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ - -v /opt/metagpt/workspace:/app/metagpt/workspace \ - metagpt/metagpt:latest - -docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" ``` -`docker run ...`做了以下事情: +详细的安装请安装 [docker_install](docs/install/docker_install_cn.md) -- 以特权模式运行，有权限运行浏览器 -- 将主机文件 `/opt/metagpt/config/key.yaml` 映射到容器文件 `/app/metagpt/config/key.yaml` -- 将主机目录 `/opt/metagpt/workspace` 映射到容器目录 `/app/metagpt/workspace` -- 执行示例命令 `python startup.py "Write a cli snake game"` +### 快速开始的演示视频 +- 在 [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) 上进行体验 +- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) +- [官方演示视频](https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d) -### 自己构建镜像 +## 教程 +- [在线文档]() +- [如何使用](docs/tutorial/usage_cn.md) +- [MetaGPT的能力及应用场景](docs/tutorial/what_can_this_do.md) +- 如何构建你自己得智能体？ + - [MetaGPT的使用和开发教程](https://deepwisdom.feishu.cn/wiki/RUnswqUIPimRJmkkDZ7cLYwOndg#Yu2AdUvymoo67Jxbp0bcu8G4nEb) +- [贡献](docs/develop/contribution.md) + - 开发者RFC + - [开发路线图](docs/ROADMAP.md) +- [样例](docs/examples/README.md) + - 调研员 + - 狼人杀游戏 +- [常见问题解答](docs/tutorial/faq.md) +- [已生成项目的展示墙](https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419) -```bash -# 您也可以自己构建metagpt镜像 -git clone https://github.com/geekan/MetaGPT.git -cd MetaGPT && docker build -t metagpt:custom . -``` +## 支持 -## 配置 +### 加入我们 -- 在 `config/key.yaml / config/config.yaml / env` 中配置您的 `OPENAI_API_KEY` -- 优先级顺序：`config/key.yaml > config/config.yaml > env` +📢 加入我们的[Discord频道](https://discord.gg/ZRHeExS6xv)！ -```bash -# 复制配置文件并进行必要的修改 -cp config/config.yaml config/key.yaml -``` +期待在那里与您相见！🎉 -| 变量名 | config/key.yaml | env | -| ----------------------------------- | ----------------------------------------- | ----------------------------------------------- | -| OPENAI_API_KEY # 用您自己的密钥替换 | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | -| OPENAI_API_BASE # 可选 | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | - -## 示例：启动一个创业公司 - -```shell -python startup.py "写一个命令行贪吃蛇" -# 开启code review模式会花费更多的金钱, 但是会提升代码质量和成功率 -python startup.py "写一个命令行贪吃蛇" --code_review True -``` - -运行脚本后，您可以在 `workspace/` 目录中找到您的新项目。 -### 平台或工具的倾向性 -可以在阐述需求时说明想要使用的平台或工具。 -例如： -```shell -python startup.py "写一个基于pygame的命令行贪吃蛇" -``` - -### 使用 - -``` -名称 - startup.py - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 - -概要 - startup.py IDEA - -描述 - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 - -位置参数 - IDEA - 类型: str - 您的创新想法，例如"写一个命令行贪吃蛇。" - -标志 - --investment=INVESTMENT - 类型: float - 默认值: 3.0 - 作为投资者，您有机会向这家AI公司投入一定的美元金额。 - --n_round=N_ROUND - 类型: int - 默认值: 5 - -备注 - 您也可以用`标志`的语法，来处理`位置参数` -``` - -### 代码实现 - -```python -from metagpt.software_company import SoftwareCompany -from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer - -async def startup(idea: str, investment: float = 3.0, n_round: int = 5): - """运行一个创业公司。做一个老板""" - company = SoftwareCompany() - company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) -``` - -你可以查看`examples`，其中有单角色（带知识库）的使用例子与仅LLM的使用例子。 - -## 快速体验 -对一些用户来说，安装配置本地环境是有困难的，下面这些教程能够让你快速体验到MetaGPT的魅力。 - -- [MetaGPT快速体验](https://deepwisdom.feishu.cn/wiki/Q8ycw6J9tiNXdHk66MRcIN8Pnlg) - -可直接在Huggingface Space体验 - -- https://huggingface.co/spaces/deepwisdom/MetaGPT - -## 联系信息 +### 联系信息 如果您对这个项目有任何问题或反馈，欢迎联系我们。我们非常欢迎您的建议！ @@ -222,13 +116,17 @@ ## 联系信息 我们会在2-3个工作日内回复所有问题。 -## 演示 +## 引用 -https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d +引用 [Arxiv paper](https://arxiv.org/abs/2308.00352): -## 加入我们 - -📢 加入我们的Discord频道！ -https://discord.gg/ZRHeExS6xv - -期待在那里与您相见！🎉 +```bibtex +@misc{hong2023metagpt, + title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, + author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu}, + year={2023}, + eprint={2308.00352}, + archivePrefix={arXiv}, + primaryClass={cs.AI} +} +``` diff --git a/docs/examples/README.md b/docs/examples/README.md new file mode 100644 index 000000000..e69de29bb From 7d28509edccbd28db957c803447a19e4500262f9 Mon Sep 17 00:00:00 2001 From: better629 Date: Thu, 2 Nov 2023 21:27:52 +0800 Subject: [PATCH 004/135] update examples README --- docs/examples/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/examples/README.md b/docs/examples/README.md index e69de29bb..4fe4628e2 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -0,0 +1,12 @@ +## Examples +We introduce some example agents created recently and show how to created these agents under the framework. + +- Researcher + - With the given research topic, the agent will derive the keywords from the topic and then search the related documents using search engine api. + - The search result will be ranked and filtered to get high quality candidates. + - Summary the final content using the candidate documents as context. +- Invoice OCR and QA + - With the given one or multi invoices, the agent can recognize the text from the image or pdf. + - Organize the text with particular structure, generally, it will be saved in a csv. + - It can answer your question like `what's the total reimbursement of Alice?` +- Werewolf Game From 250fe83de9deecebecdec1d90b873cfeddeff3fb Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Mon, 6 Nov 2023 14:37:29 +0800 Subject: [PATCH 005/135] redefine react and provide multiple react modes --- examples/build_customized_agent.py | 38 ++-------- metagpt/roles/invoice_ocr_assistant.py | 26 +------ metagpt/roles/researcher.py | 19 +---- metagpt/roles/role.py | 98 ++++++++++++++++++++++---- 4 files changed, 94 insertions(+), 87 deletions(-) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index 87d7a9c76..2bc9e31e5 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -75,10 +75,9 @@ class SimpleCoder(Role): logger.info(f"{self._setting}: ready to {self._rc.todo}") todo = self._rc.todo - msg = self._rc.memory.get()[-1] # retrieve the latest memory - instruction = msg.content + msg = self.get_memories(k=1)[0] # find the most recent messages - code_text = await SimpleWriteCode().run(instruction) + code_text = await SimpleWriteCode().run(msg.content) msg = Message(content=code_text, role=self.profile, cause_by=todo) return msg @@ -92,43 +91,20 @@ class RunnableCoder(Role): ): super().__init__(name, profile, **kwargs) self._init_actions([SimpleWriteCode, SimpleRunCode]) - - async def _think(self) -> None: - if self._rc.todo is None: - self._set_state(0) - return - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None + self._set_react_mode(react_mode="by_order") async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") todo = self._rc.todo - msg = self._rc.memory.get()[-1] - if isinstance(todo, SimpleWriteCode): - instruction = msg.content - result = await SimpleWriteCode().run(instruction) + msg = self.get_memories(k=1)[0] # find the most k recent messages + result = await todo.run(msg.content) - elif isinstance(todo, SimpleRunCode): - code_text = msg.content - result = await SimpleRunCode().run(code_text) - - msg = Message(content=result, role=self.profile, cause_by=todo) + msg = Message(content=result, role=self.profile, cause_by=type(todo)) self._rc.memory.add(msg) return msg - async def _react(self) -> Message: - while True: - await self._think() - if self._rc.todo is None: - break - await self._act() - return Message(content="All job done", role=self.profile) - -def main(msg="write a function that calculates the sum of a list"): +def main(msg="write a function that calculates the product of a list"): # role = SimpleCoder() role = RunnableCoder() logger.info(msg) diff --git a/metagpt/roles/invoice_ocr_assistant.py b/metagpt/roles/invoice_ocr_assistant.py index c307b20c0..15f831c97 100644 --- a/metagpt/roles/invoice_ocr_assistant.py +++ b/metagpt/roles/invoice_ocr_assistant.py @@ -42,17 +42,7 @@ class InvoiceOCRAssistant(Role): self.filename = "" self.origin_query = "" self.orc_data = None - - async def _think(self) -> None: - """Determine the next action to be taken by the role.""" - if self._rc.todo is None: - self._set_state(0) - return - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None + self._set_react_mode(react_mode="by_order") async def _act(self) -> Message: """Perform an action as determined by the role. @@ -94,17 +84,3 @@ class InvoiceOCRAssistant(Role): msg = Message(content=content, instruct_content=resp) self._rc.memory.add(msg) return msg - - async def _react(self) -> Message: - """Execute the invoice ocr assistant's think and actions. - - Returns: - A message containing the final result of the assistant's actions. - """ - while True: - await self._think() - if self._rc.todo is None: - break - msg = await self._act() - return msg - diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index acb46c718..c5512121a 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -31,20 +31,11 @@ class Researcher(Role): ): super().__init__(name, profile, goal, constraints, **kwargs) self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)]) + self._set_react_mode(react_mode="by_order") self.language = language if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") - async def _think(self) -> None: - if self._rc.todo is None: - self._set_state(0) - return - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None - async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") todo = self._rc.todo @@ -73,12 +64,8 @@ class Researcher(Role): self._rc.memory.add(ret) return ret - async def _react(self) -> Message: - while True: - await self._think() - if self._rc.todo is None: - break - msg = await self._act() + async def react(self) -> Message: + msg = await super().react() report = msg.instruct_content self.write_report(report.topic, report.content) return msg diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 44bb3e976..88b98f4b4 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -7,7 +7,7 @@ """ from __future__ import annotations -from typing import Iterable, Type +from typing import Iterable, Type, Union from pydantic import BaseModel, Field @@ -27,12 +27,15 @@ Please note that only the text between the first and second "===" is information {history} === -You can now choose one of the following stages to decide the stage you need to go in the next step: +Your previous stage: {previous_state} + +Now choose one of the following stages you need to go to in the next step: {states} Just answer a number between 0-{n_states}, choose the most suitable stage according to the understanding of the conversation. Please note that the answer only needs a number, no need to add any other text. -If there is no conversation record, choose 0. +If there is no conversation record or your previous stage is None, choose 0. +If you think you have completed your goal and don't need to go to any of the stages, return -1. Do not answer anything else, and do not add any other information in your answer. """ @@ -67,7 +70,7 @@ class RoleContext(BaseModel): env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) - state: int = Field(default=0) + state: int = Field(default=None) todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) @@ -100,6 +103,9 @@ class Role: self._actions = [] self._role_id = str(self._setting) self._rc = RoleContext() + # see `_set_react_mode` function for definitions of the following two attributes + self.react_mode = "react" + self.max_react_loop = 1 def _reset(self): self._states = [] @@ -116,17 +122,37 @@ class Role: self._actions.append(i) self._states.append(f"{idx}. {action}") + def _set_react_mode(self, react_mode: str, max_react_loop: int = 1): + """Set strategy of the Role reacting to observed Message. Variation lies in how + this Role elects action to perform during the _think stage, especially if it is capable of multiple Actions. + + Args: + react_mode (str): Mode for choosing action during the _think stage, can be one of + "by_order": switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...; + "react": standard think-act loop in the ReAct paper, alternating thinking and acting to solve the task, i.e. _think -> _act -> _think -> _act -> ... + Use llm to select actions in _think dynamically; + "plan_and_act": first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... + Use llm to come up with the plan dynamically. + Defaults to "by_order". + max_react_loop (int): Maximum react cycles to execute, used to prevent the agent from reacting forever. + Take effect only when react_mode is react, in which we use llm to choose actions, including termination. + Defaults to 1, i.e. _think -> _act (-> return result and end) + """ + self.react_mode = react_mode + if react_mode == "react": + self.max_react_loop = max_react_loop + def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors""" self._rc.watch.update(actions) # check RoleContext after adding watch actions self._rc.check(self._role_id) - def _set_state(self, state): + def _set_state(self, state: Union[int, None]): """Update the current state.""" self._rc.state = state logger.debug(self._actions) - self._rc.todo = self._actions[self._rc.state] + self._rc.todo = self._actions[self._rc.state] if state is not None else None def set_env(self, env: 'Environment'): """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" @@ -151,13 +177,19 @@ class Role: return prompt = self._get_prefix() prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1) + n_states=len(self._states) - 1, previous_state=self._rc.state) + # print(prompt) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") - if not next_state.isdigit() or int(next_state) not in range(len(self._states)): + if not next_state.isdigit() or int(next_state) not in range(-1, len(self._states)): logger.warning(f'Invalid answer of state, {next_state=}') - next_state = "0" - self._set_state(int(next_state)) + next_state = None + else: + next_state = int(next_state) + if next_state == -1: + logger.info(f"End actions with {next_state=}") + next_state = None + self._set_state(next_state) async def _act(self) -> Message: # prompt = self.get_prefix() @@ -203,10 +235,42 @@ class Role: self._rc.env.publish_message(msg) async def _react(self) -> Message: - """Think first, then act""" - await self._think() - logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") - return await self._act() + """Think first, then act, until the Role _think it is time to stop and requires no more todo. + This is the standard think-act loop in the ReAct paper, which alternates thinking and acting in task solving, i.e. _think -> _act -> _think -> _act -> ... + Use llm to select actions in _think dynamically + """ + actions_taken = 0 + while actions_taken < self.max_react_loop: + # think + await self._think() + if self._rc.todo is None: + break + # act + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + rsp = await self._act() + actions_taken += 1 + return rsp # return output from the last action + + async def _act_by_order(self) -> Message: + """switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...""" + for i in range(len(self._states)): + self._set_state(i) + rsp = await self._act() + return rsp # return output from the last action + + async def _plan_and_act(self) -> Message: + """first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... Use llm to come up with the plan dynamically.""" + # TODO: to be implemented + return Message("") + + async def react(self) -> Message: + """Entry to one of three strategies by which Role reacts to the observed Message""" + if self.react_mode == "react": + return await self._react() + elif self.react_mode == "by_order": + return await self._act_by_order() + elif self.react_mode == "plan_and_act": + return await self._plan_and_act() def recv(self, message: Message) -> None: """add message to history.""" @@ -223,6 +287,10 @@ class Role: return await self._react() + def get_memories(self, k=0) -> list[Message]: + """A wrapper to return the most recent k memories of this role, return all when k=0""" + return self._rc.memory.get(k=k) + async def run(self, message=None): """Observe, and think and act based on the results of the observation""" if message: @@ -237,7 +305,7 @@ class Role: logger.debug(f"{self._setting}: no news. waiting.") return - rsp = await self._react() + rsp = await self.react() # Publish the reply to the environment, waiting for the next subscriber to process self._publish_message(rsp) return rsp From c84f021467ccaf2385a4b1a95a837b9f63624ee8 Mon Sep 17 00:00:00 2001 From: Sirui Hong <34952977+stellaHSR@users.noreply.github.com> Date: Tue, 7 Nov 2023 20:45:14 +0800 Subject: [PATCH 006/135] Update README.md update paper citation --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 70460ceb4..34fc970df 100644 --- a/README.md +++ b/README.md @@ -299,8 +299,8 @@ ## Citation ```bibtex @misc{hong2023metagpt, - title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, - author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu}, + title={MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework}, + author={Sirui Hong and Mingchen Zhuge and Jonathan Chen and Xiawu Zheng and Yuheng Cheng and Ceyao Zhang and Jinlin Wang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu and Jürgen Schmidhuber}, year={2023}, eprint={2308.00352}, archivePrefix={arXiv}, From 0a57bad7425cfeb5ce22159eba2f2da9d7c95e9b Mon Sep 17 00:00:00 2001 From: Sirui Hong <34952977+stellaHSR@users.noreply.github.com> Date: Tue, 7 Nov 2023 20:46:59 +0800 Subject: [PATCH 007/135] Update README_JA.md update paper citation --- docs/README_JA.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README_JA.md b/docs/README_JA.md index 2b2c35a62..cfacff0b9 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -299,8 +299,8 @@ ## 引用 ```bibtex @misc{hong2023metagpt, - title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, - author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu}, + title={MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework}, + author={Sirui Hong and Mingchen Zhuge and Jonathan Chen and Xiawu Zheng and Yuheng Cheng and Ceyao Zhang and Jinlin Wang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu and Jürgen Schmidhuber}, year={2023}, eprint={2308.00352}, archivePrefix={arXiv}, From c6350efd7f9df1409454d3c3bd56039885d0964a Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Wed, 8 Nov 2023 14:20:12 +0800 Subject: [PATCH 008/135] default state to -1, mv react_mode to rc, use enum --- examples/build_customized_agent.py | 2 +- metagpt/roles/role.py | 61 ++++++++++++++++++------------ 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index 2bc9e31e5..c7069b768 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -104,7 +104,7 @@ class RunnableCoder(Role): self._rc.memory.add(msg) return msg -def main(msg="write a function that calculates the product of a list"): +def main(msg="write a function that calculates the product of a list and run it"): # role = SimpleCoder() role = RunnableCoder() logger.info(msg) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 88b98f4b4..0251176f7 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -8,6 +8,7 @@ from __future__ import annotations from typing import Iterable, Type, Union +from enum import Enum from pydantic import BaseModel, Field @@ -34,7 +35,6 @@ Now choose one of the following stages you need to go to in the next step: Just answer a number between 0-{n_states}, choose the most suitable stage according to the understanding of the conversation. Please note that the answer only needs a number, no need to add any other text. -If there is no conversation record or your previous stage is None, choose 0. If you think you have completed your goal and don't need to go to any of the stages, return -1. Do not answer anything else, and do not add any other information in your answer. """ @@ -49,6 +49,14 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi {name}: {result} """ +class RoleReactMode(str, Enum): + REACT = "react" + BY_ORDER = "by_order" + PLAN_AND_ACT = "plan_and_act" + + @classmethod + def values(cls): + return [item.value for item in cls] class RoleSetting(BaseModel): """Role Settings""" @@ -70,10 +78,12 @@ class RoleContext(BaseModel): env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) - state: int = Field(default=None) + state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) + react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes + max_react_loop: int = 1 class Config: arbitrary_types_allowed = True @@ -103,9 +113,6 @@ class Role: self._actions = [] self._role_id = str(self._setting) self._rc = RoleContext() - # see `_set_react_mode` function for definitions of the following two attributes - self.react_mode = "react" - self.max_react_loop = 1 def _reset(self): self._states = [] @@ -127,20 +134,21 @@ class Role: this Role elects action to perform during the _think stage, especially if it is capable of multiple Actions. Args: - react_mode (str): Mode for choosing action during the _think stage, can be one of - "by_order": switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...; - "react": standard think-act loop in the ReAct paper, alternating thinking and acting to solve the task, i.e. _think -> _act -> _think -> _act -> ... + react_mode (str): Mode for choosing action during the _think stage, can be one of: + "react": standard think-act loop in the ReAct paper, alternating thinking and acting to solve the task, i.e. _think -> _act -> _think -> _act -> ... Use llm to select actions in _think dynamically; + "by_order": switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...; "plan_and_act": first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... Use llm to come up with the plan dynamically. - Defaults to "by_order". + Defaults to "react". max_react_loop (int): Maximum react cycles to execute, used to prevent the agent from reacting forever. Take effect only when react_mode is react, in which we use llm to choose actions, including termination. Defaults to 1, i.e. _think -> _act (-> return result and end) """ - self.react_mode = react_mode - if react_mode == "react": - self.max_react_loop = max_react_loop + assert react_mode in RoleReactMode.values(), f"react_mode must be one of {RoleReactMode.values()}" + self._rc.react_mode = react_mode + if react_mode == RoleReactMode.REACT: + self._rc.max_react_loop = max_react_loop def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors""" @@ -148,11 +156,11 @@ class Role: # check RoleContext after adding watch actions self._rc.check(self._role_id) - def _set_state(self, state: Union[int, None]): + def _set_state(self, state: int): """Update the current state.""" self._rc.state = state logger.debug(self._actions) - self._rc.todo = self._actions[self._rc.state] if state is not None else None + self._rc.todo = self._actions[self._rc.state] if state >= 0 else None def set_env(self, env: 'Environment'): """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" @@ -181,14 +189,14 @@ class Role: # print(prompt) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") - if not next_state.isdigit() or int(next_state) not in range(-1, len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}') - next_state = None + if (not next_state.isdigit() and next_state != "-1") \ + or int(next_state) not in range(-1, len(self._states)): + logger.warning(f'Invalid answer of state, {next_state=}, will be set to -1') + next_state = -1 else: next_state = int(next_state) if next_state == -1: logger.info(f"End actions with {next_state=}") - next_state = None self._set_state(next_state) async def _act(self) -> Message: @@ -240,7 +248,8 @@ class Role: Use llm to select actions in _think dynamically """ actions_taken = 0 - while actions_taken < self.max_react_loop: + rsp = Message("No actions taken yet") # will be overwritten after Role _act + while actions_taken < self._rc.max_react_loop: # think await self._think() if self._rc.todo is None: @@ -265,12 +274,14 @@ class Role: async def react(self) -> Message: """Entry to one of three strategies by which Role reacts to the observed Message""" - if self.react_mode == "react": - return await self._react() - elif self.react_mode == "by_order": - return await self._act_by_order() - elif self.react_mode == "plan_and_act": - return await self._plan_and_act() + if self._rc.react_mode == RoleReactMode.REACT: + rsp = await self._react() + elif self._rc.react_mode == RoleReactMode.BY_ORDER: + rsp = await self._act_by_order() + elif self._rc.react_mode == RoleReactMode.PLAN_AND_ACT: + rsp = await self._plan_and_act() + self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None + return rsp def recv(self, message: Message) -> None: """add message to history.""" From 56c2a162acc4221c1966f41dcc4f2d54549b26a2 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Wed, 8 Nov 2023 19:42:30 +0800 Subject: [PATCH 009/135] change PROJECT_ROOT setting & SoftwareCompany -> Team --- README.md | 4 +- docs/README_CN.md | 4 +- docs/README_JA.md | 4 +- examples/debate.py | 99 +++++++----------------- metagpt/const.py | 8 +- metagpt/roles/engineer.py | 1 + metagpt/{software_company.py => team.py} | 10 +-- startup.py | 4 +- tests/metagpt/roles/test_ui.py | 4 +- tests/metagpt/test_software_company.py | 6 +- 10 files changed, 54 insertions(+), 90 deletions(-) rename metagpt/{software_company.py => team.py} (79%) diff --git a/README.md b/README.md index 70460ceb4..1171ab83c 100644 --- a/README.md +++ b/README.md @@ -270,12 +270,12 @@ ### Usage ### Code walkthrough ```python -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer async def startup(idea: str, investment: float = 3.0, n_round: int = 5): """Run a startup. Be a boss.""" - company = SoftwareCompany() + company = Team() company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) company.invest(investment) company.start_project(idea) diff --git a/docs/README_CN.md b/docs/README_CN.md index 9d6f34c11..3aa0cb05d 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -190,12 +190,12 @@ ### 使用 ### 代码实现 ```python -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer async def startup(idea: str, investment: float = 3.0, n_round: int = 5): """运行一个创业公司。做一个老板""" - company = SoftwareCompany() + company = Team() company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) company.invest(investment) company.start_project(idea) diff --git a/docs/README_JA.md b/docs/README_JA.md index 2b2c35a62..538208631 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -270,12 +270,12 @@ ### 使用方法 ### コードウォークスルー ```python -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer async def startup(idea: str, investment: float = 3.0, n_round: int = 5): """スタートアップを実行する。ボスになる。""" - company = SoftwareCompany() + company = Team() company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) company.invest(investment) company.start_project(idea) diff --git a/examples/debate.py b/examples/debate.py index 05db28070..4db7567f0 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -7,14 +7,14 @@ import asyncio import platform import fire -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.actions import Action, BossRequirement from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger -class ShoutOut(Action): - """Action: Shout out loudly in a debate (quarrel)""" +class SpeakAloud(Action): + """Action: Speak out aloud in a debate (quarrel)""" PROMPT_TEMPLATE = """ ## BACKGROUND @@ -27,7 +27,7 @@ class ShoutOut(Action): craft a strong and emotional response in 80 words, in {name}'s rhetoric and viewpoints, your will argue: """ - def __init__(self, name="ShoutOut", context=None, llm=None): + def __init__(self, name="SpeakAloud", context=None, llm=None): super().__init__(name, context, llm) async def run(self, context: str, name: str, opponent_name: str): @@ -39,96 +39,55 @@ class ShoutOut(Action): return rsp -class Trump(Role): +class Debator(Role): def __init__( self, - name: str = "Trump", - profile: str = "Republican", + name: str, + profile: str, + opponent_name: str, **kwargs, ): super().__init__(name, profile, **kwargs) - self._init_actions([ShoutOut]) - self._watch([ShoutOut]) - self.name = "Trump" - self.opponent_name = "Biden" + self._init_actions([SpeakAloud]) + self._watch([BossRequirement, SpeakAloud]) + self.name = name + self.opponent_name = opponent_name async def _observe(self) -> int: await super()._observe() # accept messages sent (from opponent) to self, disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] + self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] return len(self._rc.news) async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") + todo = self._rc.todo # An instance of SpeakAloud - msg_history = self._rc.memory.get_by_actions([ShoutOut]) - context = [] - for m in msg_history: - context.append(str(m)) - context = "\n".join(context) + memories = self.get_memories() + context = "\n".join(f"{msg.sent_from}: {msg.content}" for msg in memories) + # print(context) - rsp = await ShoutOut().run(context=context, name=self.name, opponent_name=self.opponent_name) + rsp = await todo.run(context=context, name=self.name, opponent_name=self.opponent_name) msg = Message( content=rsp, role=self.profile, - cause_by=ShoutOut, + cause_by=type(todo), sent_from=self.name, send_to=self.opponent_name, ) return msg -class Biden(Role): - def __init__( - self, - name: str = "Biden", - profile: str = "Democrat", - **kwargs, - ): - super().__init__(name, profile, **kwargs) - self._init_actions([ShoutOut]) - self._watch([BossRequirement, ShoutOut]) - self.name = "Biden" - self.opponent_name = "Trump" - - async def _observe(self) -> int: - await super()._observe() - # accept the very first human instruction (the debate topic) or messages sent (from opponent) to self, - # disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.cause_by == BossRequirement or msg.send_to == self.name] - return len(self._rc.news) - - async def _act(self) -> Message: - logger.info(f"{self._setting}: ready to {self._rc.todo}") - - msg_history = self._rc.memory.get_by_actions([BossRequirement, ShoutOut]) - context = [] - for m in msg_history: - context.append(str(m)) - context = "\n".join(context) - - rsp = await ShoutOut().run(context=context, name=self.name, opponent_name=self.opponent_name) - - msg = Message( - content=rsp, - role=self.profile, - cause_by=ShoutOut, - sent_from=self.name, - send_to=self.opponent_name, - ) - - return msg - -async def startup(idea: str, investment: float = 3.0, n_round: int = 5, - code_review: bool = False, run_tests: bool = False): - """We reuse the startup paradigm for roles to interact with each other. - Now we run a startup of presidents and watch they quarrel. :) """ - company = SoftwareCompany() - company.hire([Biden(), Trump()]) - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) +async def debate(idea: str, investment: float = 3.0, n_round: int = 5): + """Run a team of presidents and watch they quarrel. :) """ + Biden = Debator(name="Biden", profile="Democrat", opponent_name="Trump") + Trump = Debator(name="Trump", profile="Republican", opponent_name="Biden") + team = Team() + team.hire([Biden, Trump]) + team.invest(investment) + team.start_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first + await team.run(n_round=n_round) def main(idea: str, investment: float = 3.0, n_round: int = 10): @@ -141,7 +100,7 @@ def main(idea: str, investment: float = 3.0, n_round: int = 10): """ if platform.system() == "Windows": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - asyncio.run(startup(idea, investment, n_round)) + asyncio.run(debate(idea, investment, n_round)) if __name__ == '__main__': diff --git a/metagpt/const.py b/metagpt/const.py index 7f3f87dfa..86b2b6c87 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -6,7 +6,7 @@ @File : const.py """ from pathlib import Path - +from loguru import logger def get_project_root(): """Search upwards to find the project root directory.""" @@ -17,10 +17,14 @@ def get_project_root(): or (current_path / ".project_root").exists() or (current_path / ".gitignore").exists() ): + # use metagpt with git clone will land here return current_path parent_path = current_path.parent if parent_path == current_path: - raise Exception("Project root not found.") + # use metagpt with pip install will land here + cwd = Path.cwd() + logger.info(f"RPOJECT_ROOT set to current working directory: {str(cwd)}") + return cwd current_path = parent_path diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 6d65575a8..1f6685b38 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -207,6 +207,7 @@ class Engineer(Role): async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" + logger.info(f"{self._setting}: ready to WriteCode") if self.use_code_review: return await self._act_sp_precision() return await self._act_sp() diff --git a/metagpt/software_company.py b/metagpt/team.py similarity index 79% rename from metagpt/software_company.py rename to metagpt/team.py index b2bd18c58..67d3ecec8 100644 --- a/metagpt/software_company.py +++ b/metagpt/team.py @@ -16,10 +16,10 @@ from metagpt.schema import Message from metagpt.utils.common import NoMoneyException -class SoftwareCompany(BaseModel): +class Team(BaseModel): """ - Software Company: Possesses a team, SOP (Standard Operating Procedures), and a platform for instant messaging, - dedicated to writing executable code. + Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a platform for instant messaging, + dedicated to perform any multi-agent activity, such as collaboratively writing executable code. """ environment: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) @@ -42,10 +42,10 @@ class SoftwareCompany(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea): + def start_project(self, idea, send_to: str = ""): """Start a project from publishing boss requirement.""" self.idea = idea - self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) + self.environment.publish_message(Message(role="Human", content=idea, cause_by=BossRequirement, send_to=send_to)) def _save(self): logger.info(self.json()) diff --git a/startup.py b/startup.py index e2a903c9b..e9fbf94d3 100644 --- a/startup.py +++ b/startup.py @@ -11,7 +11,7 @@ from metagpt.roles import ( ProjectManager, QaEngineer, ) -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team async def startup( @@ -23,7 +23,7 @@ async def startup( implement: bool = True, ): """Run a startup. Be a boss.""" - company = SoftwareCompany() + company = Team() company.hire( [ ProductManager(), diff --git a/tests/metagpt/roles/test_ui.py b/tests/metagpt/roles/test_ui.py index 285bff323..d58d31bd9 100644 --- a/tests/metagpt/roles/test_ui.py +++ b/tests/metagpt/roles/test_ui.py @@ -2,7 +2,7 @@ # @Date : 2023/7/22 02:40 # @Author : stellahong (stellahong@fuzhi.ai) # -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.roles import ProductManager from tests.metagpt.roles.ui_role import UI @@ -15,7 +15,7 @@ def test_add_ui(): async def test_ui_role(idea: str, investment: float = 3.0, n_round: int = 5): """Run a startup. Be a boss.""" - company = SoftwareCompany() + company = Team() company.hire([ProductManager(), UI()]) company.invest(investment) company.start_project(idea) diff --git a/tests/metagpt/test_software_company.py b/tests/metagpt/test_software_company.py index 00538442c..4fc651f52 100644 --- a/tests/metagpt/test_software_company.py +++ b/tests/metagpt/test_software_company.py @@ -8,12 +8,12 @@ import pytest from metagpt.logs import logger -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team @pytest.mark.asyncio -async def test_software_company(): - company = SoftwareCompany() +async def test_team(): + company = Team() company.start_project("做一个基础搜索引擎，可以支持知识库") history = await company.run(n_round=5) logger.info(history) From 0bf2ce707e6fdf65a883da9dbf82d9bb2682f337 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Thu, 9 Nov 2023 01:41:24 +0800 Subject: [PATCH 010/135] allow human to play any roles --- metagpt/llm.py | 1 + metagpt/provider/human_gpt.py | 35 +++++++++++++++++++++++++++++++++++ metagpt/roles/role.py | 15 ++++++++++----- 3 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 metagpt/provider/human_gpt.py diff --git a/metagpt/llm.py b/metagpt/llm.py index e6f815950..8b4a13838 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -8,6 +8,7 @@ from metagpt.provider.anthropic_api import Claude2 as Claude from metagpt.provider.openai_api import OpenAIGPTAPI as LLM +from metagpt.provider.human_gpt import HumanGPT DEFAULT_LLM = LLM() CLAUDE_LLM = Claude() diff --git a/metagpt/provider/human_gpt.py b/metagpt/provider/human_gpt.py new file mode 100644 index 000000000..bc4d6dc6e --- /dev/null +++ b/metagpt/provider/human_gpt.py @@ -0,0 +1,35 @@ +''' +Filename: MetaGPT/metagpt/provider/human_speaker.py +Created Date: Wednesday, November 8th 2023, 11:55:46 pm +Author: garylin2099 +''' +from typing import Optional +from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.logs import logger + +class HumanGPT(BaseGPTAPI): + """A dummy GPT that actually takes in human input as its response. + This enables replacing LLM anywhere in the framework with a human, thus introducing human interaction + """ + + def ask(self, msg: str) -> str: + logger.info("It's your turn, please type in your response. You may also refer to the context below") + rsp = input(msg) + if rsp in ["exit", "quit"]: + exit() + return rsp + + async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: + return self.ask(msg) + + def completion(self, messages: list[dict]): + """dummy implementation of abstract method in base""" + return [] + + async def acompletion(self, messages: list[dict]): + """dummy implementation of abstract method in base""" + return [] + + async def acompletion_text(self, messages: list[dict], stream=False) -> str: + """dummy implementation of abstract method in base""" + return [] diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 0251176f7..fecca0beb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -15,7 +15,7 @@ from pydantic import BaseModel, Field # from metagpt.environment import Environment from metagpt.config import CONFIG from metagpt.actions import Action, ActionOutput -from metagpt.llm import LLM +from metagpt.llm import LLM, HumanGPT from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory from metagpt.schema import Message @@ -65,6 +65,7 @@ class RoleSetting(BaseModel): goal: str constraints: str desc: str + is_human: bool def __str__(self): return f"{self.name}({self.profile})" @@ -106,9 +107,10 @@ class RoleContext(BaseModel): class Role: """Role/Agent""" - def __init__(self, name="", profile="", goal="", constraints="", desc=""): - self._llm = LLM() - self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) + def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False): + self._llm = LLM() if not is_human else HumanGPT() + self._setting = RoleSetting(name=name, profile=profile, goal=goal, + constraints=constraints, desc=desc, is_human=is_human) self._states = [] self._actions = [] self._role_id = str(self._setting) @@ -122,8 +124,11 @@ class Role: self._reset() for idx, action in enumerate(actions): if not isinstance(action, Action): - i = action("") + i = action("", llm=self._llm) else: + if self._setting.is_human and not isinstance(action.llm, HumanGPT): + logger.warning(f"is_human attribute does not take effect," + f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") i = action i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) From a5bc55f18a6cab172e10ad77270ae9ed1d1f7e5a Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Thu, 9 Nov 2023 16:41:24 +0800 Subject: [PATCH 011/135] add deprecated warnings --- metagpt/software_company.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 metagpt/software_company.py diff --git a/metagpt/software_company.py b/metagpt/software_company.py new file mode 100644 index 000000000..d44a0068a --- /dev/null +++ b/metagpt/software_company.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/12 00:30 +@Author : alexanderwu +@File : software_company.py +""" +from metagpt.team import Team as SoftwareCompany + +import warnings +warnings.warn("metagpt.software_company is deprecated and will be removed in the future" + "Please use metagpt.team instead. SoftwareCompany class is now named as Team.", + DeprecationWarning, 2) From f0098f0ae9fef395b3842f15c8410eb29e6a916b Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Thu, 9 Nov 2023 16:51:20 +0800 Subject: [PATCH 012/135] add PROJECT_ROOT logging info --- metagpt/const.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/const.py b/metagpt/const.py index 86b2b6c87..407ce803a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -18,12 +18,13 @@ def get_project_root(): or (current_path / ".gitignore").exists() ): # use metagpt with git clone will land here + logger.info(f"PROJECT_ROOT set to {str(current_path)}") return current_path parent_path = current_path.parent if parent_path == current_path: # use metagpt with pip install will land here cwd = Path.cwd() - logger.info(f"RPOJECT_ROOT set to current working directory: {str(cwd)}") + logger.info(f"PROJECT_ROOT set to current working directory: {str(cwd)}") return cwd current_path = parent_path From 771a750e344bfd3e4f18388195a66d3aabae4a8f Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Fri, 10 Nov 2023 14:56:31 +0800 Subject: [PATCH 013/135] release 0.3.0 --- metagpt/utils/mermaid.py | 5 ++++- setup.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 5e5b275b0..204c22c67 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -34,7 +34,10 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, engine = CONFIG.mermaid_engine.lower() if engine == "nodejs": if check_cmd_exists(CONFIG.mmdc) != 0: - logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") + logger.warning( + "RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc," + "or consider changing MERMAID_ENGINE to `playwright`, `pyppeteer`, or `ink`." + ) return -1 for suffix in ["pdf", "svg", "png"]: diff --git a/setup.py b/setup.py index f9ae768e6..239156ae3 100644 --- a/setup.py +++ b/setup.py @@ -30,16 +30,16 @@ with open(path.join(here, "requirements.txt"), encoding="utf-8") as f: setup( name="metagpt", - version="0.1", + version="0.3.0", description="The Multi-Role Meta Programming Framework", long_description=long_description, long_description_content_type="text/markdown", - url="https://gitlab.deepwisdomai.com/pub/metagpt", + url="https://github.com/geekan/MetaGPT", author="Alexander Wu", author_email="alexanderwu@fuzhi.ai", license="Apache 2.0", keywords="metagpt multi-role multi-agent programming gpt llm", - packages=find_packages(exclude=["contrib", "docs", "examples"]), + packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, extras_require={ From 7bf808ee258fe15ab08ad234a081ef1660548833 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Fri, 17 Nov 2023 00:20:46 +0800 Subject: [PATCH 014/135] renaming & provide example & small bug fix --- examples/build_customized_agent.py | 17 +- examples/build_customized_multi_agents.py | 150 ++++++++++++++++++ examples/debate.py | 2 + metagpt/llm.py | 2 +- .../{human_gpt.py => human_provider.py} | 6 +- metagpt/roles/role.py | 6 +- 6 files changed, 164 insertions(+), 19 deletions(-) create mode 100644 examples/build_customized_multi_agents.py rename metagpt/provider/{human_gpt.py => human_provider.py} (85%) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index c7069b768..ed00e8320 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -19,15 +19,6 @@ class SimpleWriteCode(Action): PROMPT_TEMPLATE = """ Write a python function that can {instruction} and provide two runnnable test cases. Return ```python your_code_here ``` with NO other texts, - example: - ```python - # function - def add(a, b): - return a + b - # test cases - print(add(1, 2)) - print(add(3, 4)) - ``` your code: """ @@ -73,12 +64,12 @@ class SimpleCoder(Role): async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - todo = self._rc.todo + todo = self._rc.todo # todo will be SimpleWriteCode() msg = self.get_memories(k=1)[0] # find the most recent messages - code_text = await SimpleWriteCode().run(msg.content) - msg = Message(content=code_text, role=self.profile, cause_by=todo) + code_text = await todo.run(msg.content) + msg = Message(content=code_text, role=self.profile, cause_by=type(todo)) return msg @@ -95,6 +86,8 @@ class RunnableCoder(Role): async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") + # By choosing the Action by order under the hood + # todo will be first SimpleWriteCode() then SimpleRunCode() todo = self._rc.todo msg = self.get_memories(k=1)[0] # find the most k recent messages diff --git a/examples/build_customized_multi_agents.py b/examples/build_customized_multi_agents.py new file mode 100644 index 000000000..02221e8e9 --- /dev/null +++ b/examples/build_customized_multi_agents.py @@ -0,0 +1,150 @@ +''' +Filename: MetaGPT/examples/build_customized_multi_agents.py +Created Date: Wednesday, November 15th 2023, 7:12:39 pm +Author: garylin2099 +''' +import re +import asyncio +import fire +from metagpt.actions import Action, BossRequirement +from metagpt.roles import Role +from metagpt.team import Team +from metagpt.schema import Message +from metagpt.logs import logger + +def parse_code(rsp): + pattern = r'```python(.*)```' + match = re.search(pattern, rsp, re.DOTALL) + code_text = match.group(1) if match else rsp + return code_text + +class SimpleWriteCode(Action): + + PROMPT_TEMPLATE = """ + Write a python function that can {instruction}. + Return ```python your_code_here ``` with NO other texts, + your code: + """ + + def __init__(self, name="SimpleWriteCode", context=None, llm=None): + super().__init__(name, context, llm) + + async def run(self, instruction: str): + + prompt = self.PROMPT_TEMPLATE.format(instruction=instruction) + + rsp = await self._aask(prompt) + + code_text = parse_code(rsp) + + return code_text + +class SimpleCoder(Role): + def __init__( + self, + name: str = "Alice", + profile: str = "SimpleCoder", + **kwargs, + ): + super().__init__(name, profile, **kwargs) + self._watch([BossRequirement]) + self._init_actions([SimpleWriteCode]) + +class SimpleWriteTest(Action): + + PROMPT_TEMPLATE = """ + Context: {context} + Write {k} unit tests using pytest for the given function, assuming you have imported it. + Return ```python your_code_here ``` with NO other texts, + your code: + """ + + def __init__(self, name="SimpleWriteTest", context=None, llm=None): + super().__init__(name, context, llm) + + async def run(self, context: str, k: int = 3): + + prompt = self.PROMPT_TEMPLATE.format(context=context, k=k) + + rsp = await self._aask(prompt) + + code_text = parse_code(rsp) + + return code_text + +class SimpleTester(Role): + def __init__( + self, + name: str = "Bob", + profile: str = "SimpleTester", + **kwargs, + ): + super().__init__(name, profile, **kwargs) + self._init_actions([SimpleWriteTest]) + # self._watch([SimpleWriteCode]) + self._watch([SimpleWriteCode, SimpleWriteReview]) # feel free to try this too + + async def _act(self) -> Message: + logger.info(f"{self._setting}: ready to {self._rc.todo}") + todo = self._rc.todo + + # context = self.get_memories(k=1)[0].content # use the most recent memory as context + context = self.get_memories() # use all memories as context + + code_text = await todo.run(context, k=5) # specify arguments + msg = Message(content=code_text, role=self.profile, cause_by=type(todo)) + + return msg + +class SimpleWriteReview(Action): + + PROMPT_TEMPLATE = """ + Context: {context} + Review the test cases and provide one critical comments: + """ + + def __init__(self, name="SimpleWriteReview", context=None, llm=None): + super().__init__(name, context, llm) + + async def run(self, context: str): + + prompt = self.PROMPT_TEMPLATE.format(context=context) + + rsp = await self._aask(prompt) + + return rsp + +class SimpleReviewer(Role): + def __init__( + self, + name: str = "Charlie", + profile: str = "SimpleReviewer", + **kwargs, + ): + super().__init__(name, profile, **kwargs) + self._init_actions([SimpleWriteReview]) + self._watch([SimpleWriteTest]) + +async def main( + idea: str = "write a function that calculates the product of a list", + investment: float = 3.0, + n_round: int = 5, + add_human: bool = False, +): + logger.info(idea) + + team = Team() + team.hire( + [ + SimpleCoder(), + SimpleTester(), + SimpleReviewer(is_human=add_human), + ] + ) + + team.invest(investment=investment) + team.start_project(idea) + await team.run(n_round=n_round) + +if __name__ == '__main__': + fire.Fire(main) diff --git a/examples/debate.py b/examples/debate.py index 4db7567f0..a37e60848 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -77,6 +77,8 @@ class Debator(Role): send_to=self.opponent_name, ) + self._rc.memory.add(msg) + return msg async def debate(idea: str, investment: float = 3.0, n_round: int = 5): diff --git a/metagpt/llm.py b/metagpt/llm.py index 8b4a13838..9324da126 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -8,7 +8,7 @@ from metagpt.provider.anthropic_api import Claude2 as Claude from metagpt.provider.openai_api import OpenAIGPTAPI as LLM -from metagpt.provider.human_gpt import HumanGPT +from metagpt.provider.human_provider import HumanProvider DEFAULT_LLM = LLM() CLAUDE_LLM = Claude() diff --git a/metagpt/provider/human_gpt.py b/metagpt/provider/human_provider.py similarity index 85% rename from metagpt/provider/human_gpt.py rename to metagpt/provider/human_provider.py index bc4d6dc6e..1d12f972f 100644 --- a/metagpt/provider/human_gpt.py +++ b/metagpt/provider/human_provider.py @@ -1,5 +1,5 @@ ''' -Filename: MetaGPT/metagpt/provider/human_speaker.py +Filename: MetaGPT/metagpt/provider/human_provider.py Created Date: Wednesday, November 8th 2023, 11:55:46 pm Author: garylin2099 ''' @@ -7,8 +7,8 @@ from typing import Optional from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.logs import logger -class HumanGPT(BaseGPTAPI): - """A dummy GPT that actually takes in human input as its response. +class HumanProvider(BaseGPTAPI): + """Humans provide themselves as a 'model', which actually takes in human input as its response. This enables replacing LLM anywhere in the framework with a human, thus introducing human interaction """ diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index fecca0beb..b96c361c0 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -15,7 +15,7 @@ from pydantic import BaseModel, Field # from metagpt.environment import Environment from metagpt.config import CONFIG from metagpt.actions import Action, ActionOutput -from metagpt.llm import LLM, HumanGPT +from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory from metagpt.schema import Message @@ -108,7 +108,7 @@ class Role: """Role/Agent""" def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False): - self._llm = LLM() if not is_human else HumanGPT() + self._llm = LLM() if not is_human else HumanProvider() self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, is_human=is_human) self._states = [] @@ -126,7 +126,7 @@ class Role: if not isinstance(action, Action): i = action("", llm=self._llm) else: - if self._setting.is_human and not isinstance(action.llm, HumanGPT): + if self._setting.is_human and not isinstance(action.llm, HumanProvider): logger.warning(f"is_human attribute does not take effect," f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") i = action From 5c66b29a80e538cb5db0e11b5844c335c55c3864 Mon Sep 17 00:00:00 2001 From: Sirui Hong <34952977+stellaHSR@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:56:29 +0800 Subject: [PATCH 015/135] Update README.md rm as waitlist --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index d2ca26cdd..1307fbb53 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ # MetaGPT: The Multi-Agent Framework

- AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces Hugging Face From aa14c6664bddc91742de7a30dc44657eb4a517f8 Mon Sep 17 00:00:00 2001 From: Sirui Hong <34952977+stellaHSR@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:56:59 +0800 Subject: [PATCH 016/135] Update README_CN.md rm as waitlist --- docs/README_CN.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/README_CN.md b/docs/README_CN.md index 3aa0cb05d..a49ca9b77 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -19,7 +19,6 @@ # MetaGPT: 多智能体框架

- AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces Hugging Face From f8ebfa9a742abf3f0b187428a6c14d328c8c1460 Mon Sep 17 00:00:00 2001 From: Sirui Hong <34952977+stellaHSR@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:57:28 +0800 Subject: [PATCH 017/135] Update README_JA.md rm as waitlist --- docs/README_JA.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/README_JA.md b/docs/README_JA.md index 1ced60b83..cba84df80 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -19,7 +19,6 @@ # MetaGPT: マルチエージェントフレームワーク

- AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces Hugging Face From 705a3e962b2d5a7d8bb643901b84f0d9b34870e2 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Fri, 17 Nov 2023 14:40:07 +0800 Subject: [PATCH 018/135] formatting --- examples/build_customized_agent.py | 9 +++++++-- examples/build_customized_multi_agents.py | 14 +++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index ed00e8320..be34e5e5e 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -9,6 +9,7 @@ import asyncio import fire +from metagpt.llm import LLM from metagpt.actions import Action from metagpt.roles import Role from metagpt.schema import Message @@ -22,7 +23,7 @@ class SimpleWriteCode(Action): your code: """ - def __init__(self, name="SimpleWriteCode", context=None, llm=None): + def __init__(self, name: str = "SimpleWriteCode", context=None, llm: LLM = None): super().__init__(name, context, llm) async def run(self, instruction: str): @@ -42,8 +43,9 @@ class SimpleWriteCode(Action): code_text = match.group(1) if match else rsp return code_text + class SimpleRunCode(Action): - def __init__(self, name="SimpleRunCode", context=None, llm=None): + def __init__(self, name: str = "SimpleRunCode", context=None, llm: LLM = None): super().__init__(name, context, llm) async def run(self, code_text: str): @@ -52,6 +54,7 @@ class SimpleRunCode(Action): logger.info(f"{code_result=}") return code_result + class SimpleCoder(Role): def __init__( self, @@ -73,6 +76,7 @@ class SimpleCoder(Role): return msg + class RunnableCoder(Role): def __init__( self, @@ -97,6 +101,7 @@ class RunnableCoder(Role): self._rc.memory.add(msg) return msg + def main(msg="write a function that calculates the product of a list and run it"): # role = SimpleCoder() role = RunnableCoder() diff --git a/examples/build_customized_multi_agents.py b/examples/build_customized_multi_agents.py index 02221e8e9..0df927e32 100644 --- a/examples/build_customized_multi_agents.py +++ b/examples/build_customized_multi_agents.py @@ -6,6 +6,8 @@ Author: garylin2099 import re import asyncio import fire + +from metagpt.llm import LLM from metagpt.actions import Action, BossRequirement from metagpt.roles import Role from metagpt.team import Team @@ -26,7 +28,7 @@ class SimpleWriteCode(Action): your code: """ - def __init__(self, name="SimpleWriteCode", context=None, llm=None): + def __init__(self, name: str = "SimpleWriteCode", context=None, llm: LLM = None): super().__init__(name, context, llm) async def run(self, instruction: str): @@ -39,6 +41,7 @@ class SimpleWriteCode(Action): return code_text + class SimpleCoder(Role): def __init__( self, @@ -50,6 +53,7 @@ class SimpleCoder(Role): self._watch([BossRequirement]) self._init_actions([SimpleWriteCode]) + class SimpleWriteTest(Action): PROMPT_TEMPLATE = """ @@ -59,7 +63,7 @@ class SimpleWriteTest(Action): your code: """ - def __init__(self, name="SimpleWriteTest", context=None, llm=None): + def __init__(self, name: str = "SimpleWriteTest", context=None, llm: LLM = None): super().__init__(name, context, llm) async def run(self, context: str, k: int = 3): @@ -72,6 +76,7 @@ class SimpleWriteTest(Action): return code_text + class SimpleTester(Role): def __init__( self, @@ -96,6 +101,7 @@ class SimpleTester(Role): return msg + class SimpleWriteReview(Action): PROMPT_TEMPLATE = """ @@ -103,7 +109,7 @@ class SimpleWriteReview(Action): Review the test cases and provide one critical comments: """ - def __init__(self, name="SimpleWriteReview", context=None, llm=None): + def __init__(self, name: str = "SimpleWriteReview", context=None, llm: LLM = None): super().__init__(name, context, llm) async def run(self, context: str): @@ -114,6 +120,7 @@ class SimpleWriteReview(Action): return rsp + class SimpleReviewer(Role): def __init__( self, @@ -125,6 +132,7 @@ class SimpleReviewer(Role): self._init_actions([SimpleWriteReview]) self._watch([SimpleWriteTest]) + async def main( idea: str = "write a function that calculates the product of a list", investment: float = 3.0, From 0153bb5416042bf449a2885b4f255627abf45402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Sat, 18 Nov 2023 16:09:56 +0800 Subject: [PATCH 019/135] feat: add ask code via function. --- metagpt/provider/base_gpt_api.py | 20 ++++++++-- metagpt/provider/openai_api.py | 55 +++++++++++++++++++++++++-- metagpt/utils/function_schema.py | 29 ++++++++++++++ tests/metagpt/provider/test_openai.py | 22 +++++++++++ 4 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 metagpt/utils/function_schema.py create mode 100644 tests/metagpt/provider/test_openai.py diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index de61167b9..2cfc3fa1f 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : base_gpt_api.py """ +import json from abc import abstractmethod from typing import Optional @@ -14,7 +15,8 @@ from metagpt.provider.base_chatbot import BaseChatbot class BaseGPTAPI(BaseChatbot): """GPT API abstract class, requiring all inheritors to provide a series of standard capabilities""" - system_prompt = 'You are a helpful assistant.' + + system_prompt = "You are a helpful assistant." def _user_msg(self, msg: str) -> dict[str, str]: return {"role": "user", "content": msg} @@ -108,11 +110,23 @@ class BaseGPTAPI(BaseChatbot): """Required to provide the first text of choice""" return rsp.get("choices")[0]["message"]["content"] + def get_choice_function(self, rsp: dict) -> dict: + """Required to provide the first function of choice. for example: + "function": { + "name": "execute", + "arguments": "{\n \"language\": \"python\",\n \"code\": \"print('Hello, World!')\"\n}" + } + """ + return rsp.get("choices")[0]["message"]["tool_calls"][0]["function"].to_dict() + + def get_choice_function_arguments(self, rsp: dict) -> dict: + """Required to provide the first function arguments of choice.""" + return json.loads(self.get_choice_function(rsp)["arguments"]) + def messages_to_prompt(self, messages: list[dict]): """[{"role": "user", "content": msg}] to user: etc.""" - return '\n'.join([f"{i['role']}: {i['content']}" for i in messages]) + return "\n".join([f"{i['role']}: {i['content']}" for i in messages]) def messages_to_dict(self, messages): """objects to [{"role": "user", "content": msg}] etc.""" return [i.to_dict() for i in messages] - \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 6ebed2c16..b9698e77d 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -21,6 +21,7 @@ from tenacity import ( from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.utils.function_schema import general_function_schema, general_tool_choice from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( TOKEN_COSTS, @@ -110,7 +111,6 @@ class CostManager(metaclass=Singleton): """ return self.total_completion_tokens - def get_total_cost(self): """ Get the total cost of API calls. @@ -120,7 +120,6 @@ class CostManager(metaclass=Singleton): """ return self.total_cost - def get_costs(self) -> Costs: """Get all costs""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) @@ -181,7 +180,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self._update_costs(usage) return full_reply_content - def _cons_kwargs(self, messages: list[dict]) -> dict: + def _cons_kwargs(self, messages: list[dict], **configs) -> dict: kwargs = { "messages": messages, "max_tokens": self.get_max_tokens(messages), @@ -190,6 +189,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): "temperature": 0.3, "timeout": 3, } + if configs: + kwargs.update(configs) + if CONFIG.openai_api_type == "azure": if CONFIG.deployment_name and CONFIG.deployment_id: raise ValueError("You can only use one of the `deployment_id` or `deployment_name` model") @@ -239,6 +241,53 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): rsp = await self._achat_completion(messages) return self.get_choice_text(rsp) + def _func_configs(self, messages: list[dict], **kwargs) -> dict: + if "tools" not in kwargs: + configs = { + "tools": [{"type": "function", "function": general_function_schema}], + "tool_choice": general_tool_choice, + } + kwargs.update(configs) + + return self._cons_kwargs(messages, **kwargs) + + def _chat_completion_function(self, messages: list[dict], **kwargs) -> dict: + rsp = self.llm.ChatCompletion.create(**self._func_configs(messages, **kwargs)) + self._update_costs(rsp.get("usage")) + return rsp + + async def _achat_completion_function(self, messages: list[dict], **chat_configs) -> dict: + rsp = await self.llm.ChatCompletion.acreate(**self._func_configs(messages, **chat_configs)) + self._update_costs(rsp.get("usage")) + return rsp + + def ask_code(self, messages: list[dict], **kwargs) -> dict: + """Use function of tools to ask a code. + https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools + + Examples: + + >>> llm = OpenAIGPTAPI() + >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}] + >>> llm.ask_code(msg) + {'language': 'python', 'code': "print('Hello, World!')"} + """ + rsp = self._chat_completion_function(messages, **kwargs) + return self.get_choice_function_arguments(rsp) + + async def aask_code(self, messages: list[dict], **kwargs) -> dict: + """Use function of tools to ask a code. + https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools + + Examples: + + >>> llm = OpenAIGPTAPI() + >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}] + >>> rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} + """ + rsp = await self._achat_completion_function(messages, **kwargs) + return self.get_choice_function_arguments(rsp) + def _calc_usage(self, messages: list[dict], rsp: str) -> dict: usage = {} if CONFIG.calc_usage: diff --git a/metagpt/utils/function_schema.py b/metagpt/utils/function_schema.py new file mode 100644 index 000000000..9d7cef927 --- /dev/null +++ b/metagpt/utils/function_schema.py @@ -0,0 +1,29 @@ +# function in tools, https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools +general_function_schema = { + "name": "execute", + "description": "Executes code on the user's machine, **in the users local environment**, and returns the output", + "parameters": { + "type": "object", + "properties": { + "language": { + "type": "string", + "description": "The programming language (required parameter to the `execute` function)", + "enum": [ + "python", + "R", + "shell", + "applescript", + "javascript", + "html", + "powershell", + ], + }, + "code": {"type": "string", "description": "The code to execute (required)"}, + }, + "required": ["language", "code"], + }, +} + +# tool_choice value for general_function_schema +# https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice +general_tool_choice = {"type": "function", "function": {"name": "execute"}} diff --git a/tests/metagpt/provider/test_openai.py b/tests/metagpt/provider/test_openai.py new file mode 100644 index 000000000..4cbc896e0 --- /dev/null +++ b/tests/metagpt/provider/test_openai.py @@ -0,0 +1,22 @@ +import pytest + +from metagpt.provider.openai_api import OpenAIGPTAPI + + +@pytest.mark.asyncio +async def test_aask_code(): + llm = OpenAIGPTAPI() + msg = [{"role": "user", "content": "Write a python hello world code."}] + rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} + assert "language" in rsp + assert "code" in rsp + assert len(rsp["code"]) > 0 + + +def test_ask_code(): + llm = OpenAIGPTAPI() + msg = [{"role": "user", "content": "Write a python hello world code."}] + rsp = llm.ask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} + assert "language" in rsp + assert "code" in rsp + assert len(rsp["code"]) > 0 From aca50086c3e51c153cf24a777bd9af53030a8ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Sat, 18 Nov 2023 16:59:14 +0800 Subject: [PATCH 020/135] chore: add comment for general_function_schema. --- metagpt/utils/function_schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/utils/function_schema.py b/metagpt/utils/function_schema.py index 9d7cef927..f9b84fc60 100644 --- a/metagpt/utils/function_schema.py +++ b/metagpt/utils/function_schema.py @@ -1,4 +1,5 @@ # function in tools, https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools +# Reference: https://github.com/KillianLucas/open-interpreter/blob/v0.1.14/interpreter/llm/setup_openai_coding_llm.py general_function_schema = { "name": "execute", "description": "Executes code on the user's machine, **in the users local environment**, and returns the output", From 2a881589330a86c3c7c7fdd5d608d20679bc6cc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Sat, 18 Nov 2023 18:14:38 +0800 Subject: [PATCH 021/135] chore: remove and renamed function_schema.py --- metagpt/{utils/function_schema.py => provider/constant.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename metagpt/{utils/function_schema.py => provider/constant.py} (100%) diff --git a/metagpt/utils/function_schema.py b/metagpt/provider/constant.py similarity index 100% rename from metagpt/utils/function_schema.py rename to metagpt/provider/constant.py From 218eeef4b8616a47b190531db34cbd5cee26e57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Sat, 18 Nov 2023 18:15:50 +0800 Subject: [PATCH 022/135] chore: messages supports more types. --- metagpt/provider/openai_api.py | 22 +++++++++++++-- tests/metagpt/provider/test_openai.py | 39 +++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index b9698e77d..287668c83 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -21,7 +21,8 @@ from tenacity import ( from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.utils.function_schema import general_function_schema, general_tool_choice +from metagpt.provider.constant import general_function_schema, general_tool_choice +from metagpt.schema import Message from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( TOKEN_COSTS, @@ -261,7 +262,22 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self._update_costs(rsp.get("usage")) return rsp - def ask_code(self, messages: list[dict], **kwargs) -> dict: + def _process_message(self, messages: Union[str, Message, list[dict]]) -> list[dict]: + """convert messages to list[dict].""" + if isinstance(messages, list): + return messages + + if isinstance(messages, Message): + messages = [messages.to_dict()] + elif isinstance(messages, str): + messages = [{"role": "user", "content": messages}] + else: + raise ValueError( + f"Only support messages type are: str, Message, list[dict], but got {type(messages).__name__}!" + ) + return messages + + def ask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: """Use function of tools to ask a code. https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools @@ -272,6 +288,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): >>> llm.ask_code(msg) {'language': 'python', 'code': "print('Hello, World!')"} """ + messages = self._process_message(messages) rsp = self._chat_completion_function(messages, **kwargs) return self.get_choice_function_arguments(rsp) @@ -285,6 +302,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}] >>> rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} """ + messages = self._process_message(messages) rsp = await self._achat_completion_function(messages, **kwargs) return self.get_choice_function_arguments(rsp) diff --git a/tests/metagpt/provider/test_openai.py b/tests/metagpt/provider/test_openai.py index 4cbc896e0..d6d9f4f9d 100644 --- a/tests/metagpt/provider/test_openai.py +++ b/tests/metagpt/provider/test_openai.py @@ -1,6 +1,7 @@ import pytest from metagpt.provider.openai_api import OpenAIGPTAPI +from metagpt.schema import UserMessage @pytest.mark.asyncio @@ -13,6 +14,26 @@ async def test_aask_code(): assert len(rsp["code"]) > 0 +@pytest.mark.asyncio +async def test_aask_code_str(): + llm = OpenAIGPTAPI() + msg = "Write a python hello world code." + rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} + assert "language" in rsp + assert "code" in rsp + assert len(rsp["code"]) > 0 + + +@pytest.mark.asyncio +async def test_aask_code_Message(): + llm = OpenAIGPTAPI() + msg = UserMessage("Write a python hello world code.") + rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} + assert "language" in rsp + assert "code" in rsp + assert len(rsp["code"]) > 0 + + def test_ask_code(): llm = OpenAIGPTAPI() msg = [{"role": "user", "content": "Write a python hello world code."}] @@ -20,3 +41,21 @@ def test_ask_code(): assert "language" in rsp assert "code" in rsp assert len(rsp["code"]) > 0 + + +def test_ask_code_str(): + llm = OpenAIGPTAPI() + msg = "Write a python hello world code." + rsp = llm.ask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} + assert "language" in rsp + assert "code" in rsp + assert len(rsp["code"]) > 0 + + +def test_ask_code_Message(): + llm = OpenAIGPTAPI() + msg = UserMessage("Write a python hello world code.") + rsp = llm.ask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} + assert "language" in rsp + assert "code" in rsp + assert len(rsp["code"]) > 0 From 62254a184e485ab5bb5219b84c40660b56b216a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Sat, 18 Nov 2023 18:33:00 +0800 Subject: [PATCH 023/135] chore: add examples for ask_code and aask_code. --- metagpt/provider/openai_api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 287668c83..8f8d45d73 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -284,6 +284,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): Examples: >>> llm = OpenAIGPTAPI() + >>> llm.ask_code("Write a python hello world code.") + {'language': 'python', 'code': "print('Hello, World!')"} >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}] >>> llm.ask_code(msg) {'language': 'python', 'code': "print('Hello, World!')"} @@ -292,13 +294,16 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): rsp = self._chat_completion_function(messages, **kwargs) return self.get_choice_function_arguments(rsp) - async def aask_code(self, messages: list[dict], **kwargs) -> dict: + async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: """Use function of tools to ask a code. https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools Examples: >>> llm = OpenAIGPTAPI() + >>> rsp = await llm.ask_code("Write a python hello world code.") + >>> rsp + {'language': 'python', 'code': "print('Hello, World!')"} >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}] >>> rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} """ From 96ce036bd4b6fd01402a11e1e9f5abb274e93e77 Mon Sep 17 00:00:00 2001 From: better629 Date: Sat, 18 Nov 2023 21:34:14 +0800 Subject: [PATCH 024/135] add zhipuai api with extra async invoke methods --- metagpt/config.py | 9 +- metagpt/llm.py | 26 ++-- metagpt/provider/zhipuai/__init__.py | 3 + metagpt/provider/zhipuai/async_sse_client.py | 77 ++++++++++ metagpt/provider/zhipuai/zhipu_model_api.py | 76 ++++++++++ metagpt/provider/zhipuai_api.py | 139 +++++++++++++++++++ metagpt/utils/token_counter.py | 6 +- 7 files changed, 323 insertions(+), 13 deletions(-) create mode 100644 metagpt/provider/zhipuai/__init__.py create mode 100644 metagpt/provider/zhipuai/async_sse_client.py create mode 100644 metagpt/provider/zhipuai/zhipu_model_api.py create mode 100644 metagpt/provider/zhipuai_api.py diff --git a/metagpt/config.py b/metagpt/config.py index 27455d38d..3f9e742bd 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -45,10 +45,11 @@ class Config(metaclass=Singleton): self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") - if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key - ): - raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") + self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY") + if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and \ + (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) and \ + (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key): + raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy if openai_proxy: diff --git a/metagpt/llm.py b/metagpt/llm.py index e6f815950..1f6a6bb1a 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -6,14 +6,24 @@ @File : llm.py """ +from metagpt.logs import logger +from metagpt.config import CONFIG from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM +from metagpt.provider.openai_api import OpenAIGPTAPI +from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +from metagpt.provider.spark_api import SparkAPI -DEFAULT_LLM = LLM() -CLAUDE_LLM = Claude() -async def ai_func(prompt): - """使用LLM进行QA - QA with LLMs - """ - return await DEFAULT_LLM.aask(prompt) +def LLM(): + """ initialize different LLM instance according to the key field existence""" + # TODO a little trick, can use registry to initialize LLM instance further + if CONFIG.openai_api_key and CONFIG.openai_api_key.starswith("sk-"): + llm = OpenAIGPTAPI() + elif CONFIG.claude_api_key: + llm = Claude() + elif CONFIG.spark_api_key: + llm = SparkAPI() + elif CONFIG.zhipuai_api_key: + llm = ZhiPuAIGPTAPI() + + return llm diff --git a/metagpt/provider/zhipuai/__init__.py b/metagpt/provider/zhipuai/__init__.py new file mode 100644 index 000000000..2bcf8efd0 --- /dev/null +++ b/metagpt/provider/zhipuai/__init__.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : diff --git a/metagpt/provider/zhipuai/async_sse_client.py b/metagpt/provider/zhipuai/async_sse_client.py new file mode 100644 index 000000000..7a4275982 --- /dev/null +++ b/metagpt/provider/zhipuai/async_sse_client.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : async_sse_client to make keep the use of Event to access response + +from zhipuai.utils.sse_client import SSEClient, Event, _FIELD_SEPARATOR + + +class AsyncSSEClient(SSEClient): + + async def _aread(self): + data = b"" + async for chunk in self._event_source: + for line in chunk.splitlines(True): + data += line + if data.endswith((b"\r\r", b"\n\n", b"\r\n\r\n")): + yield data + data = b"" + if data: + yield data + + async def async_events(self): + async for chunk in self._aread(): + event = Event() + # Split before decoding so splitlines() only uses \r and \n + for line in chunk.splitlines(): + # Decode the line. + line = line.decode(self._char_enc) + + # Lines starting with a separator are comments and are to be + # ignored. + if not line.strip() or line.startswith(_FIELD_SEPARATOR): + continue + + data = line.split(_FIELD_SEPARATOR, 1) + field = data[0] + + # Ignore unknown fields. + if field not in event.__dict__: + self._logger.debug( + "Saw invalid field %s while parsing " "Server Side Event", field + ) + continue + + if len(data) > 1: + # From the spec: + # "If value starts with a single U+0020 SPACE character, + # remove it from value." + if data[1].startswith(" "): + value = data[1][1:] + else: + value = data[1] + else: + # If no value is present after the separator, + # assume an empty value. + value = "" + + # The data field may come over multiple lines and their values + # are concatenated with each other. + if field == "data": + event.__dict__[field] += value + "\n" + else: + event.__dict__[field] = value + + # Events with no data are not dispatched. + if not event.data: + continue + + # If the data field ends with a newline, remove it. + if event.data.endswith("\n"): + event.data = event.data[0:-1] + + # Empty event names default to 'message' + event.event = event.event or "message" + + # Dispatch the event + self._logger.debug("Dispatching %s...", event) + yield event diff --git a/metagpt/provider/zhipuai/zhipu_model_api.py b/metagpt/provider/zhipuai/zhipu_model_api.py new file mode 100644 index 000000000..f1fd6f2e2 --- /dev/null +++ b/metagpt/provider/zhipuai/zhipu_model_api.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : zhipu model api to support sync & async for invoke & sse_invoke + +import zhipuai +from zhipuai.model_api.api import ModelAPI, InvokeType +from zhipuai.utils.http_client import headers as zhipuai_default_headers +from zhipuai.utils.sse_client import SSEClient + +from metagpt.provider.zhipuai.async_sse_client import AsyncSSEClient +from metagpt.provider.general_api_requestor import GeneralAPIRequestor + + +class ZhiPuModelAPI(ModelAPI): + + @classmethod + def get_header(cls) -> dict: + token = cls._generate_token() + zhipuai_default_headers.update({"Authorization": token}) + return zhipuai_default_headers + + @classmethod + def get_sse_header(cls) -> dict: + token = cls._generate_token() + headers = { + "Authorization": token + } + return headers + + @classmethod + def split_zhipu_api_url(cls, invoke_type: InvokeType, kwargs): + # use this method to prevent zhipu api upgrading to different version. + zhipu_api_url = cls._build_api_url(kwargs, invoke_type) + # example: https://open.bigmodel.cn/api/paas/v3/model-api/{model}/{invoke_method} + arr = zhipu_api_url.split("/api/") + # ("https://open.bigmodel.cn/api/" , "/paas/v3/model-api/chatglm_turbo/invoke") + return f"{arr[0]}/api", f"/{arr[1]}" + + @classmethod + async def arequest(cls, invoke_type: InvokeType, stream: bool, method: str, headers: dict, kwargs): + # TODO to make the async request to be more generic for models in http mode. + assert method in ["post", "get"] + + api_base, url = cls.split_zhipu_api_url(invoke_type, kwargs) + requester = GeneralAPIRequestor(api_base=api_base) + result, _, api_key = await requester.arequest( + method=method, + url=url, + headers=headers, + stream=stream, + params=kwargs, + request_timeout=zhipuai.api_timeout_seconds + ) + + return result + + @classmethod + async def ainvoke(cls, **kwargs) -> dict: + """ async invoke different from raw method `async_invoke` which get the final result by task_id""" + headers = cls.get_header() + resp = await cls.arequest(invoke_type=InvokeType.SYNC, + stream=False, + method="post", + headers=headers, + kwargs=kwargs) + return resp + + @classmethod + async def asse_invoke(cls, **kwargs) -> AsyncSSEClient: + """ async sse_invoke """ + headers = cls.get_sse_header() + return AsyncSSEClient(await cls.arequest(invoke_type=InvokeType.SSE, + stream=True, + method="post", + headers=headers, + kwargs=kwargs)) diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py new file mode 100644 index 000000000..4e8e6b760 --- /dev/null +++ b/metagpt/provider/zhipuai_api.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : zhipuai LLM from https://open.bigmodel.cn/dev/api#sdk + +from enum import Enum +import json +from tenacity import ( + after_log, + retry, + retry_if_exception_type, + stop_after_attempt, + wait_fixed, +) +from requests import ConnectionError + +import zhipuai + +from metagpt.config import CONFIG +from metagpt.logs import logger +from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.provider.openai_api import CostManager, log_and_reraise +from metagpt.provider.zhipuai.zhipu_model_api import ZhiPuModelAPI +from metagpt.utils.ahttp_client import astream + + +class ZhiPuEvent(Enum): + ADD = "add" + ERROR = "error" + INTERRUPTED = "interrupted" + FINISH = "finish" + + +class ZhiPuAIGPTAPI(BaseGPTAPI): + """ + Refs to `https://open.bigmodel.cn/dev/api#chatglm_turbo` + From now, there is only one model named `chatglm_turbo` + """ + + use_system_prompt: bool = False # zhipuai has no system prompt when use api + + def __init__(self): + self.__init_zhipuai(CONFIG) + self.llm = ZhiPuModelAPI + self.model = "chatglm_turbo" # so far only one model, just use it + self._cost_manager = CostManager() + + def __init_zhipuai(self, config: CONFIG): + assert config.zhipuai_api_key + zhipuai.api_key = config.zhipuai_api_key + + def _const_kwargs(self, messages: list[dict]) -> dict: + kwargs = { + "model": self.model, + "prompt": messages, + "temperature": 0.3 + } + return kwargs + + def _update_costs(self, usage: dict): + """ update each request's token cost """ + if CONFIG.calc_usage: + try: + prompt_tokens = int(usage.get("prompt_tokens", 0)) + completion_tokens = int(usage.get("completion_tokens", 0)) + self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) + except Exception as e: + logger.error("zhipuai updats costs failed!", e) + + def get_choice_text(self, resp: dict) -> str: + """ get the first text of choice from llm response """ + assist_msg = resp.get("data").get("choices")[-1] + assert assist_msg["role"] == "assistant" + return assist_msg.get("content") + + def completion(self, messages: list[dict]) -> dict: + resp = self.llm.invoke(**self._const_kwargs(messages)) + usage = resp.get("data").get("usage") + self._update_costs(usage) + return resp + + async def _achat_completion(self, messages: list[dict]) -> dict: + resp = await self.llm.ainvoke(**self._const_kwargs(messages)) + usage = resp.get("data").get("usage") + self._update_costs(usage) + return resp + + async def acompletion(self, messages: list[dict]) -> dict: + return await self._achat_completion(messages) + + async def _achat_completion_stream(self, messages: list[dict]) -> str: + response = await self.llm.asse_invoke(**self._const_kwargs(messages)) + collected_content = [] + usage = {} + async for event in response.async_events(): + if event.event == ZhiPuEvent.ADD.value: + content = event.data + collected_content.append(content) + print(content, end="") + elif event.event == ZhiPuEvent.ERROR.value or event.event == ZhiPuEvent.INTERRUPTED.value: + content = event.data + logger.error(f"event error: {content}", end="") + collected_content.append([content]) + elif event.event == ZhiPuEvent.FINISH.value: + """ + event.meta + { + "task_status":"SUCCESS", + "usage":{ + "completion_tokens":351, + "prompt_tokens":595, + "total_tokens":946 + }, + "task_id":"xx", + "request_id":"xxx" + } + """ + meta = json.loads(event.meta) + usage = meta.get("usage") + else: + print(f"zhipuapi else event: {event.data}", end="") + + self._update_costs(usage) + full_content = "".join(collected_content) + logger.info(f"full_content: {full_content} !!") + return full_content + + # @retry( + # stop=stop_after_attempt(3), + # wait=wait_fixed(1), + # after=after_log(logger, logger.level("WARNING").name), + # retry=retry_if_exception_type(ConnectionError), + # retry_error_callback=log_and_reraise + # ) + async def acompletion_text(self, messages: list[dict], stream=False) -> str: + """ response in async with stream or non-stream mode """ + if stream: + return await self._achat_completion_stream(messages) + resp = await self._achat_completion(messages) + return self.get_choice_text(resp) diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index a5a65803a..1af96f272 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -22,6 +22,7 @@ TOKEN_COSTS = { "gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12}, "gpt-4-0613": {"prompt": 0.06, "completion": 0.12}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, + "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069} # 32k version, prompt + completion tokens=0.005￥/k-tokens } @@ -37,6 +38,7 @@ TOKEN_MAX = { "gpt-4-32k-0314": 32768, "gpt-4-0613": 8192, "text-embedding-ada-002": 8192, + "chatglm_turbo": 32768 } @@ -68,7 +70,9 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): return count_message_tokens(messages, model="gpt-4-0613") else: raise NotImplementedError( - f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""" + f"num_tokens_from_messages() is not implemented for model {model}. " + f"See https://github.com/openai/openai-python/blob/main/chatml.md " + f"for information on how messages are converted to tokens." ) num_tokens = 0 for message in messages: From 66f27ca2d599c7e6420ac11d4f44c2d4668d5d28 Mon Sep 17 00:00:00 2001 From: better629 Date: Sat, 18 Nov 2023 21:35:41 +0800 Subject: [PATCH 025/135] add General Async API for http-based LLM model --- metagpt/provider/general_api_requestor.py | 64 +++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 metagpt/provider/general_api_requestor.py diff --git a/metagpt/provider/general_api_requestor.py b/metagpt/provider/general_api_requestor.py new file mode 100644 index 000000000..e4e5f0f96 --- /dev/null +++ b/metagpt/provider/general_api_requestor.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : General Async API for http-based LLM model + +from typing import AsyncGenerator, Tuple, Union, Optional, Literal +import aiohttp +import asyncio + +from openai.api_requestor import APIRequestor, aiohttp_session + +from metagpt.logs import logger + + +class GeneralAPIRequestor(APIRequestor): + """ + usage + # full_url = "{api_base}{url}" + requester = GeneralAPIRequestor(api_base=api_base) + result, _, api_key = await requester.arequest( + method=method, + url=url, + headers=headers, + stream=stream, + params=kwargs, + request_timeout=120 + ) + """ + + def _interpret_response_line( + self, rbody: str, rcode: int, rheaders, stream: bool + ) -> str: + # just do nothing to meet the APIRequestor process and return the raw data + # due to the openai sdk will convert the data into OpenAIResponse which we don't need in general cases. + + return rbody + + async def _interpret_async_response( + self, result: aiohttp.ClientResponse, stream: bool + ) -> Tuple[Union[str, AsyncGenerator[str, None]], bool]: + if stream and "text/event-stream" in result.headers.get("Content-Type", ""): + logger.warning("stream") + return ( + self._interpret_response_line( + line, result.status, result.headers, stream=True + ) + async for line in result.content + ), True + else: + logger.warning("non stream") + try: + await result.read() + except (aiohttp.ServerTimeoutError, asyncio.TimeoutError) as e: + raise TimeoutError("Request timed out") from e + except aiohttp.ClientError as exp: + logger.warning(f"response: {result.content}, exp: {exp}") + return ( + self._interpret_response_line( + await result.read(), # let the caller to decode the msg + result.status, + result.headers, + stream=False, + ), + False, + ) From 8e201384bf67020e6cf8efcf248dc2fd71977e57 Mon Sep 17 00:00:00 2001 From: better629 Date: Sat, 18 Nov 2023 21:28:49 +0800 Subject: [PATCH 026/135] add use_system_prompt to judge if need to add system_prompt part --- config/config.yaml | 3 +++ metagpt/provider/base_chatbot.py | 1 + metagpt/provider/base_gpt_api.py | 8 +++++--- requirements.txt | 1 + 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index b2c50991d..fc6961f9e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -31,6 +31,9 @@ RPM: 10 #DEPLOYMENT_NAME: "YOUR_DEPLOYMENT_NAME" #DEPLOYMENT_ID: "YOUR_DEPLOYMENT_ID" +#### if zhipuai from `https://open.bigmodel.cn`. You can set here or export API_KEY="YOUR_API_KEY" +ZHIPUAI_API_KEY: "YOUR_API_KEY" + #### for Search ## Supported values: serpapi/google/serper/ddg diff --git a/metagpt/provider/base_chatbot.py b/metagpt/provider/base_chatbot.py index abdf423f4..72e6c94f9 100644 --- a/metagpt/provider/base_chatbot.py +++ b/metagpt/provider/base_chatbot.py @@ -13,6 +13,7 @@ from dataclasses import dataclass class BaseChatbot(ABC): """Abstract GPT class""" mode: str = "API" + use_system_prompt: bool = True @abstractmethod def ask(self, msg: str) -> str: diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index de61167b9..3a157b63e 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -32,15 +32,17 @@ class BaseGPTAPI(BaseChatbot): return self._system_msg(self.system_prompt) def ask(self, msg: str) -> str: - message = [self._default_system_msg(), self._user_msg(msg)] + message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] rsp = self.completion(message) return self.get_choice_text(rsp) async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: if system_msgs: - message = self._system_msgs(system_msgs) + [self._user_msg(msg)] + message = self._system_msgs(system_msgs) + [self._user_msg(msg)] if self.use_system_prompt \ + else [self._user_msg(msg)] else: - message = [self._default_system_msg(), self._user_msg(msg)] + message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt \ + else [self._user_msg(msg)] rsp = await self.acompletion_text(message, stream=True) logger.debug(message) # logger.debug(rsp) diff --git a/requirements.txt b/requirements.txt index 093298775..66e8dce02 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,3 +44,4 @@ ta==0.10.2 semantic-kernel==0.3.13.dev0 wrapt==1.15.0 websocket-client==0.58.0 +zhipuai==1.0.7 From 2c81cc3e0f976d9b3774761d72036c60aa824866 Mon Sep 17 00:00:00 2001 From: better629 Date: Sat, 18 Nov 2023 22:00:52 +0800 Subject: [PATCH 027/135] add zhipuai_api unittest and remove useless log --- metagpt/provider/general_api_requestor.py | 2 - metagpt/provider/zhipuai/zhipu_model_api.py | 1 - tests/metagpt/provider/test_zhipuai_api.py | 47 +++++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 tests/metagpt/provider/test_zhipuai_api.py diff --git a/metagpt/provider/general_api_requestor.py b/metagpt/provider/general_api_requestor.py index e4e5f0f96..169b7c146 100644 --- a/metagpt/provider/general_api_requestor.py +++ b/metagpt/provider/general_api_requestor.py @@ -38,7 +38,6 @@ class GeneralAPIRequestor(APIRequestor): self, result: aiohttp.ClientResponse, stream: bool ) -> Tuple[Union[str, AsyncGenerator[str, None]], bool]: if stream and "text/event-stream" in result.headers.get("Content-Type", ""): - logger.warning("stream") return ( self._interpret_response_line( line, result.status, result.headers, stream=True @@ -46,7 +45,6 @@ class GeneralAPIRequestor(APIRequestor): async for line in result.content ), True else: - logger.warning("non stream") try: await result.read() except (aiohttp.ServerTimeoutError, asyncio.TimeoutError) as e: diff --git a/metagpt/provider/zhipuai/zhipu_model_api.py b/metagpt/provider/zhipuai/zhipu_model_api.py index f1fd6f2e2..e1d52061d 100644 --- a/metagpt/provider/zhipuai/zhipu_model_api.py +++ b/metagpt/provider/zhipuai/zhipu_model_api.py @@ -5,7 +5,6 @@ import zhipuai from zhipuai.model_api.api import ModelAPI, InvokeType from zhipuai.utils.http_client import headers as zhipuai_default_headers -from zhipuai.utils.sse_client import SSEClient from metagpt.provider.zhipuai.async_sse_client import AsyncSSEClient from metagpt.provider.general_api_requestor import GeneralAPIRequestor diff --git a/tests/metagpt/provider/test_zhipuai_api.py b/tests/metagpt/provider/test_zhipuai_api.py new file mode 100644 index 000000000..6a0c70de5 --- /dev/null +++ b/tests/metagpt/provider/test_zhipuai_api.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : the unittest of ZhiPuAIGPTAPI + +import pytest + +from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI + + +default_resp = { + "code": 200, + "data": { + "choices": [ + {"role": "assistant", "content": "I'm chatglm-turbo"} + ] + } +} + +messages = [ + {"role": "user", "content": "who are you"} +] + + +def mock_llm_ask(self, messages: list[dict]) -> dict: + return default_resp + + +def test_zhipuai_completion(mocker): + mocker.patch("metagpt.provider.zhipuai_api.ZhiPuAIGPTAPI.completion", mock_llm_ask) + + resp = ZhiPuAIGPTAPI().completion(messages) + assert resp["code"] == 200 + assert "chatglm-turbo" in resp["data"]["choices"][0]["content"] + + +async def mock_llm_aask(self, messgaes: list[dict], stream: bool = False) -> dict: + return default_resp + + +@pytest.mark.asyncio +async def test_zhipuai_acompletion(mocker): + mocker.patch("metagpt.provider.zhipuai_api.ZhiPuAIGPTAPI.acompletion_text", mock_llm_aask) + + resp = await ZhiPuAIGPTAPI().acompletion_text(messages, stream=False) + + assert resp["code"] == 200 + assert "chatglm-turbo" in resp["data"]["choices"][0]["content"] From 6ef3b213c3504a3d2e5f4f5ffc04b47e70dc78fd Mon Sep 17 00:00:00 2001 From: better629 Date: Sat, 18 Nov 2023 22:17:40 +0800 Subject: [PATCH 028/135] fix small problem --- metagpt/llm.py | 2 +- metagpt/provider/zhipuai/async_sse_client.py | 1 + metagpt/provider/zhipuai/zhipu_model_api.py | 6 +++++- metagpt/provider/zhipuai_api.py | 17 ++++++++--------- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 1f6a6bb1a..e9b80d7a8 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -14,7 +14,7 @@ from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI from metagpt.provider.spark_api import SparkAPI -def LLM(): +def LLM() -> "BaseGPTAPI": """ initialize different LLM instance according to the key field existence""" # TODO a little trick, can use registry to initialize LLM instance further if CONFIG.openai_api_key and CONFIG.openai_api_key.starswith("sk-"): diff --git a/metagpt/provider/zhipuai/async_sse_client.py b/metagpt/provider/zhipuai/async_sse_client.py index 7a4275982..b819fdc63 100644 --- a/metagpt/provider/zhipuai/async_sse_client.py +++ b/metagpt/provider/zhipuai/async_sse_client.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # @Desc : async_sse_client to make keep the use of Event to access response +# refs to `https://github.com/zhipuai/zhipuai-sdk-python/blob/main/zhipuai/utils/sse_client.py` from zhipuai.utils.sse_client import SSEClient, Event, _FIELD_SEPARATOR diff --git a/metagpt/provider/zhipuai/zhipu_model_api.py b/metagpt/provider/zhipuai/zhipu_model_api.py index e1d52061d..618b2e865 100644 --- a/metagpt/provider/zhipuai/zhipu_model_api.py +++ b/metagpt/provider/zhipuai/zhipu_model_api.py @@ -29,8 +29,12 @@ class ZhiPuModelAPI(ModelAPI): @classmethod def split_zhipu_api_url(cls, invoke_type: InvokeType, kwargs): # use this method to prevent zhipu api upgrading to different version. + # and follow the GeneralAPIRequestor implemented based on openai sdk zhipu_api_url = cls._build_api_url(kwargs, invoke_type) - # example: https://open.bigmodel.cn/api/paas/v3/model-api/{model}/{invoke_method} + """ + example: + zhipu_api_url: https://open.bigmodel.cn/api/paas/v3/model-api/{model}/{invoke_method} + """ arr = zhipu_api_url.split("/api/") # ("https://open.bigmodel.cn/api/" , "/paas/v3/model-api/chatglm_turbo/invoke") return f"{arr[0]}/api", f"/{arr[1]}" diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 4e8e6b760..2ad1944c2 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -68,7 +68,7 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): def get_choice_text(self, resp: dict) -> str: """ get the first text of choice from llm response """ - assist_msg = resp.get("data").get("choices")[-1] + assist_msg = resp.get("data", {}).get("choices", [{"role": "error"}])[-1] assert assist_msg["role"] == "assistant" return assist_msg.get("content") @@ -121,16 +121,15 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): self._update_costs(usage) full_content = "".join(collected_content) - logger.info(f"full_content: {full_content} !!") return full_content - # @retry( - # stop=stop_after_attempt(3), - # wait=wait_fixed(1), - # after=after_log(logger, logger.level("WARNING").name), - # retry=retry_if_exception_type(ConnectionError), - # retry_error_callback=log_and_reraise - # ) + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + after=after_log(logger, logger.level("WARNING").name), + retry=retry_if_exception_type(ConnectionError), + retry_error_callback=log_and_reraise + ) async def acompletion_text(self, messages: list[dict], stream=False) -> str: """ response in async with stream or non-stream mode """ if stream: From ef5cb6f6deaf5f3215a895b592ce424e125ef38b Mon Sep 17 00:00:00 2001 From: better629 Date: Sun, 19 Nov 2023 15:38:47 +0800 Subject: [PATCH 029/135] update readme with online docs url --- README.md | 32 ++++++++++++++++++-------------- docs/README_CN.md | 30 ++++++++++++++++-------------- docs/examples/README.md | 12 ------------ 3 files changed, 34 insertions(+), 40 deletions(-) delete mode 100644 docs/examples/README.md diff --git a/README.md b/README.md index f3fb3dff9..60193e086 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ # MetaGPT: The Multi-Agent Framework

Software Company Multi-Role Schematic (Gradually Implementing)

+ + ## Install ### Conda installation @@ -58,7 +60,7 @@ # setup your OPENAI_API_KEY in key.yaml copy from config.yaml python3 startup.py "Write a cli snake game" ``` -detail installation please refer to [cli_install](docs/install/cli_install.md) +detail installation please refer to [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#installation-for-full-features) ### Docker installation @@ -78,28 +80,30 @@ # Step 2: Run metagpt demo with container python startup.py "Write a cli snake game" ``` -detail installation please refer to [docker_install](docs/install/docker_install.md) +detail installation please refer to [docker_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-with-docker) ### QuickStart & Demo Video - Try it on [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) - [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) - [Official Demo Video](https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d) +https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 + ## Tutorial -- [Oneline Document]() -- [Usage](docs/tutorial/usage.md) -- [What can MetaGPT do?](docs/tutorial/what_can_this_do.md) -- How to build your own agents? - - [MetaGPT Usage & Development Guide](https://deepwisdom.feishu.cn/wiki/RUnswqUIPimRJmkkDZ7cLYwOndg#Yu2AdUvymoo67Jxbp0bcu8G4nEb) -- [Contribution](docs/develop/contribution.md) - - Develop RFC +- 🗒 [Online Document](https://docs.deepwisdom.ai/) +- 💻 [Usage](https://docs.deepwisdom.ai/guide/get_started/quickstart.html) +- 🔎 [What can MetaGPT do?](https://docs.deepwisdom.ai/guide/get_started/introduction.html) +- 🛠 How to build your own agents? + - [MetaGPT Usage & Development Guide | Agent 101](https://docs.deepwisdom.ai/guide/tutorials/agent_101.html) + - [MetaGPT Usage & Development Guide | MultiAgent 101](https://docs.deepwisdom.ai/guide/tutorials/multi_agent_101.html) +- 🧑‍💻 Contribution - [Develop Roadmap](docs/ROADMAP.md) -- [Examples](docs/examples/README.md) - - Researcher - - Werewolf Game -- [FAQs](docs/tutorial/faq.md) -- [The generated projects display wall](https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419) +- 🔖 Use Cases + - [Debate](https://docs.deepwisdom.ai/guide/use_cases/multi_agent/debate.html) + - [Researcher](https://docs.deepwisdom.ai/guide/use_cases/agent/researcher.html) + - [Recepit Assistant](https://docs.deepwisdom.ai/guide/use_cases/agent/receipt_assistant.html) +- ❓ [FAQs](https://docs.deepwisdom.ai/guide/faq.html) ## Support diff --git a/docs/README_CN.md b/docs/README_CN.md index 8c04e5066..6dac0e210 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -57,7 +57,7 @@ # 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY python3 startup.py "Write a cli snake game" ``` -详细的安装请安装 [cli_install](docs/install/cli_install_cn.md) +详细的安装请安装 [cli_install](https://docs.deepwisdom.ai/zhcn/guide/get_started/installation.html#%E5%AE%89%E8%A3%85%E5%85%A8%E9%83%A8%E5%8A%9F%E8%83%BD) ### Docker安装 @@ -77,27 +77,29 @@ # 步骤2: 使用容器运行metagpt演示 python startup.py "Write a cli snake game" ``` -详细的安装请安装 [docker_install](docs/install/docker_install_cn.md) +详细的安装请安装 [docker_install](https://docs.deepwisdom.ai/zhcn/guide/get_started/installation.html#%E4%BD%BF%E7%94%A8docker%E5%AE%89%E8%A3%85) ### 快速开始的演示视频 - 在 [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) 上进行体验 - [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) - [官方演示视频](https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d) +https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 + ## 教程 -- [在线文档]() -- [如何使用](docs/tutorial/usage_cn.md) -- [MetaGPT的能力及应用场景](docs/tutorial/what_can_this_do.md) -- 如何构建你自己得智能体？ - - [MetaGPT的使用和开发教程](https://deepwisdom.feishu.cn/wiki/RUnswqUIPimRJmkkDZ7cLYwOndg#Yu2AdUvymoo67Jxbp0bcu8G4nEb) -- [贡献](docs/develop/contribution.md) - - 开发者RFC +- 🗒 [在线文档](https://docs.deepwisdom.ai/zhcn/) +- 💻 [如何使用](https://docs.deepwisdom.ai/zhcn/guide/get_started/quickstart.html) +- 🔎 [MetaGPT的能力及应用场景](https://docs.deepwisdom.ai/zhcn/guide/get_started/introduction.html) +- 🛠 如何构建你自己得智能体？ + - [MetaGPT的使用和开发教程 | 智能体入门](https://docs.deepwisdom.ai/zhcn/guide/tutorials/agent_101.html) + - [MetaGPT的使用和开发教程 | 多智能体入门](https://docs.deepwisdom.ai/zhcn/guide/tutorials/multi_agent_101.html) +- 🧑‍💻 贡献 - [开发路线图](docs/ROADMAP.md) -- [样例](docs/examples/README.md) - - 调研员 - - 狼人杀游戏 -- [常见问题解答](docs/tutorial/faq.md) -- [已生成项目的展示墙](https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419) +- 🔖 示例 + - [辩论](https://docs.deepwisdom.ai/zhcn/guide/use_cases/multi_agent/debate.html) + - [调研员](https://docs.deepwisdom.ai/zhcn/guide/use_cases/agent/researcher.html) + - [票据助手](https://docs.deepwisdom.ai/zhcn/guide/use_cases/agent/receipt_assistant.html) +- ❓ [常见问题解答](https://docs.deepwisdom.ai/zhcn/guide/faq.html) ## 支持 diff --git a/docs/examples/README.md b/docs/examples/README.md deleted file mode 100644 index 4fe4628e2..000000000 --- a/docs/examples/README.md +++ /dev/null @@ -1,12 +0,0 @@ -## Examples -We introduce some example agents created recently and show how to created these agents under the framework. - -- Researcher - - With the given research topic, the agent will derive the keywords from the topic and then search the related documents using search engine api. - - The search result will be ranked and filtered to get high quality candidates. - - Summary the final content using the candidate documents as context. -- Invoice OCR and QA - - With the given one or multi invoices, the agent can recognize the text from the image or pdf. - - Organize the text with particular structure, generally, it will be saved in a csv. - - It can answer your question like `what's the total reimbursement of Alice?` -- Werewolf Game From bf47b500bfeeb8b0c53965ce82acf717f058ab56 Mon Sep 17 00:00:00 2001 From: better629 Date: Sun, 19 Nov 2023 15:43:56 +0800 Subject: [PATCH 030/135] fix --- docs/README_CN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README_CN.md b/docs/README_CN.md index 6dac0e210..1839b8f67 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -90,11 +90,11 @@ ## 教程 - 🗒 [在线文档](https://docs.deepwisdom.ai/zhcn/) - 💻 [如何使用](https://docs.deepwisdom.ai/zhcn/guide/get_started/quickstart.html) - 🔎 [MetaGPT的能力及应用场景](https://docs.deepwisdom.ai/zhcn/guide/get_started/introduction.html) -- 🛠 如何构建你自己得智能体？ +- 🛠 如何构建你自己的智能体？ - [MetaGPT的使用和开发教程 | 智能体入门](https://docs.deepwisdom.ai/zhcn/guide/tutorials/agent_101.html) - [MetaGPT的使用和开发教程 | 多智能体入门](https://docs.deepwisdom.ai/zhcn/guide/tutorials/multi_agent_101.html) - 🧑‍💻 贡献 - - [开发路线图](docs/ROADMAP.md) + - [开发路线图](ROADMAP.md) - 🔖 示例 - [辩论](https://docs.deepwisdom.ai/zhcn/guide/use_cases/multi_agent/debate.html) - [调研员](https://docs.deepwisdom.ai/zhcn/guide/use_cases/agent/researcher.html) From 1b91e66e3aa78099c90239ba6a6f76fcfbc5292e Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 20 Nov 2023 09:23:47 +0800 Subject: [PATCH 031/135] update --- README.md | 8 ++++---- docs/README_CN.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 60193e086..c537e2a3b 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ # MetaGPT: The Multi-Agent Framework ## Install -### Conda installation +### Pip installation ```bash # Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) @@ -50,17 +50,17 @@ # conda create -n metagpt python=3.9 # conda activate metagpt python3 --version -# Step 3: Clone the repository to your local machine, and install it. +# Step 3: Clone the repository to your local machine for latest version, and install it. git clone https://github.com/geekan/metagpt cd metagpt -pip3 install -e. +pip3 install -e. # or pip3 install metagpt # for stable version # Step 4: run the startup.py # setup your OPENAI_API_KEY in key.yaml copy from config.yaml python3 startup.py "Write a cli snake game" ``` -detail installation please refer to [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#installation-for-full-features) +detail installation please refer to [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) ### Docker installation diff --git a/docs/README_CN.md b/docs/README_CN.md index 1839b8f67..3da41793e 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -34,7 +34,7 @@ # MetaGPT: 多智能体框架

软件公司多角色示意图（正在逐步实现）

## 安装 -### Conda安装 +### Pip安装 ```bash # 第 1 步：确保您的系统上安装了 NPM。并使用npm安装mermaid-js @@ -47,17 +47,17 @@ # conda create -n metagpt python=3.9 # conda activate metagpt python --version -# 第 3 步：克隆仓库到您的本地机器，并进行安装。 +# 第 3 步：克隆最新仓库到您的本地机器，并进行安装。 git clone https://github.com/geekan/metagpt cd metagpt -pip install -e. +pip install -e. # 或者 pip3 install metagpt # 安装稳定版本 # 第 4 步：执行startup.py # 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY python3 startup.py "Write a cli snake game" ``` -详细的安装请安装 [cli_install](https://docs.deepwisdom.ai/zhcn/guide/get_started/installation.html#%E5%AE%89%E8%A3%85%E5%85%A8%E9%83%A8%E5%8A%9F%E8%83%BD) +详细的安装请安装 [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) ### Docker安装 From 33ddababfff7e0d4446292d8055fc2bdfe897f7a Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 20 Nov 2023 09:27:22 +0800 Subject: [PATCH 032/135] update --- docs/README_CN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/README_CN.md b/docs/README_CN.md index 3da41793e..22e8358b2 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -45,12 +45,12 @@ # 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下 # 可以使用conda来初始化新的python环境 # conda create -n metagpt python=3.9 # conda activate metagpt -python --version +python3 --version # 第 3 步：克隆最新仓库到您的本地机器，并进行安装。 git clone https://github.com/geekan/metagpt cd metagpt -pip install -e. # 或者 pip3 install metagpt # 安装稳定版本 +pip3 install -e. # 或者 pip3 install metagpt # 安装稳定版本 # 第 4 步：执行startup.py # 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY From f8f938f333ff781ccc3755c798200df2e1272f87 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 20 Nov 2023 14:46:31 +0800 Subject: [PATCH 033/135] fix config when open llm model hosts as openai interface --- config/config.yaml | 4 ++-- metagpt/llm.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index fc6961f9e..bed67083c 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,7 +7,7 @@ ## Or, you can configure OPENAI_PROXY to access official OPENAI_API_BASE. OPENAI_API_BASE: "https://api.openai.com/v1" #OPENAI_PROXY: "http://127.0.0.1:8118" -#OPENAI_API_KEY: "YOUR_API_KEY" +#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model OPENAI_API_MODEL: "gpt-4" MAX_TOKENS: 1500 RPM: 10 @@ -32,7 +32,7 @@ RPM: 10 #DEPLOYMENT_ID: "YOUR_DEPLOYMENT_ID" #### if zhipuai from `https://open.bigmodel.cn`. You can set here or export API_KEY="YOUR_API_KEY" -ZHIPUAI_API_KEY: "YOUR_API_KEY" +# ZHIPUAI_API_KEY: "YOUR_API_KEY" #### for Search diff --git a/metagpt/llm.py b/metagpt/llm.py index e9b80d7a8..13e5a56e0 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -17,13 +17,15 @@ from metagpt.provider.spark_api import SparkAPI def LLM() -> "BaseGPTAPI": """ initialize different LLM instance according to the key field existence""" # TODO a little trick, can use registry to initialize LLM instance further - if CONFIG.openai_api_key and CONFIG.openai_api_key.starswith("sk-"): + if CONFIG.openai_api_key and CONFIG.openai_api_key.startswith("sk-"): llm = OpenAIGPTAPI() elif CONFIG.claude_api_key: llm = Claude() elif CONFIG.spark_api_key: llm = SparkAPI() - elif CONFIG.zhipuai_api_key: + elif CONFIG.zhipuai_api_key and CONFIG.zhipuai_api_key != "YOUR_API_KEY": llm = ZhiPuAIGPTAPI() + else: + raise RuntimeError("You should config a LLM configuration first") return llm From 33f0ca2aee7e90021d914b752aa9cd016f7c5a16 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 20 Nov 2023 15:04:37 +0800 Subject: [PATCH 034/135] add optional for npm --- README.md | 7 ++++--- docs/README_CN.md | 7 ++++--- docs/README_JA.md | 4 ++-- docs/install/cli_install.md | 4 ++-- docs/install/cli_install_cn.md | 4 ++-- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c537e2a3b..1929a8bdd 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ ## Install ### Pip installation ```bash -# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) +# Step 1 [Optional]: If you want to save the artifacts like PRD in the workspace, you can execute the step. By default, the framework is compatible, and the entire process can be run completely without executing this step. +# If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) npm --version sudo npm install -g @mermaid-js/mermaid-cli @@ -51,8 +52,8 @@ # conda activate metagpt python3 --version # Step 3: Clone the repository to your local machine for latest version, and install it. -git clone https://github.com/geekan/metagpt -cd metagpt +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT pip3 install -e. # or pip3 install metagpt # for stable version # Step 4: run the startup.py diff --git a/docs/README_CN.md b/docs/README_CN.md index 22e8358b2..3e0879c1b 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -37,7 +37,8 @@ ## 安装 ### Pip安装 ```bash -# 第 1 步：确保您的系统上安装了 NPM。并使用npm安装mermaid-js +# 第 1 步【可选的】：如果你想在执行过程中保存像PRD文件这些产物，可以执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。 +# 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid-js npm --version sudo npm install -g @mermaid-js/mermaid-cli @@ -48,8 +49,8 @@ # conda activate metagpt python3 --version # 第 3 步：克隆最新仓库到您的本地机器，并进行安装。 -git clone https://github.com/geekan/metagpt -cd metagpt +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT pip3 install -e. # 或者 pip3 install metagpt # 安装稳定版本 # 第 4 步：执行startup.py diff --git a/docs/README_JA.md b/docs/README_JA.md index 2b2c35a62..dfb9c8372 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -68,8 +68,8 @@ # ステップ 2: Python 3.9+ がシステムにインストールされてい python --version # ステップ 3: リポジトリをローカルマシンにクローンし、インストールする。 -git clone https://github.com/geekan/metagpt -cd metagpt +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT pip install -e. ``` diff --git a/docs/install/cli_install.md b/docs/install/cli_install.md index 19f0e3c75..80deda771 100644 --- a/docs/install/cli_install.md +++ b/docs/install/cli_install.md @@ -17,8 +17,8 @@ # Step 2: Ensure that Python 3.9+ is installed on your system. You can check thi python3 --version # Step 3: Clone the repository to your local machine, and install it. -git clone https://github.com/geekan/metagpt -cd metagpt +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT pip install -e. ``` diff --git a/docs/install/cli_install_cn.md b/docs/install/cli_install_cn.md index fe97a0b80..f351090ed 100644 --- a/docs/install/cli_install_cn.md +++ b/docs/install/cli_install_cn.md @@ -18,8 +18,8 @@ # 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下 python --version # 第 3 步：克隆仓库到您的本地机器，并进行安装。 -git clone https://github.com/geekan/metagpt -cd metagpt +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT pip install -e. ``` From 920caa048aaa85eb47503f6c48f50f0a3af16773 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 20 Nov 2023 15:11:55 +0800 Subject: [PATCH 035/135] change step order of pip-install --- README.md | 16 ++++++++-------- docs/README_CN.md | 16 ++++++++-------- docs/README_JA.md | 17 +++++++++++------ 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 1929a8bdd..0606d2be6 100644 --- a/README.md +++ b/README.md @@ -40,25 +40,25 @@ ## Install ### Pip installation ```bash -# Step 1 [Optional]: If you want to save the artifacts like PRD in the workspace, you can execute the step. By default, the framework is compatible, and the entire process can be run completely without executing this step. -# If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) -npm --version -sudo npm install -g @mermaid-js/mermaid-cli - -# Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using: +# Step 1: Ensure that Python 3.9+ is installed on your system. You can check this by using: # You can use conda to initialize a new python env # conda create -n metagpt python=3.9 # conda activate metagpt python3 --version -# Step 3: Clone the repository to your local machine for latest version, and install it. +# Step 2: Clone the repository to your local machine for latest version, and install it. git clone https://github.com/geekan/MetaGPT.git cd MetaGPT pip3 install -e. # or pip3 install metagpt # for stable version -# Step 4: run the startup.py +# Step 3: run the startup.py # setup your OPENAI_API_KEY in key.yaml copy from config.yaml python3 startup.py "Write a cli snake game" + +# Step 4 [Optional]: If you want to save the artifacts like PRD in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step. +# If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) +npm --version +sudo npm install -g @mermaid-js/mermaid-cli ``` detail installation please refer to [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) diff --git a/docs/README_CN.md b/docs/README_CN.md index 3e0879c1b..d38d6908b 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -37,25 +37,25 @@ ## 安装 ### Pip安装 ```bash -# 第 1 步【可选的】：如果你想在执行过程中保存像PRD文件这些产物，可以执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。 -# 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid-js -npm --version -sudo npm install -g @mermaid-js/mermaid-cli - -# 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： +# 第 1 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： # 可以使用conda来初始化新的python环境 # conda create -n metagpt python=3.9 # conda activate metagpt python3 --version -# 第 3 步：克隆最新仓库到您的本地机器，并进行安装。 +# 第 2 步：克隆最新仓库到您的本地机器，并进行安装。 git clone https://github.com/geekan/MetaGPT.git cd MetaGPT pip3 install -e. # 或者 pip3 install metagpt # 安装稳定版本 -# 第 4 步：执行startup.py +# 第 3 步：执行startup.py # 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY python3 startup.py "Write a cli snake game" + +# 第 4 步【可选的】：如果你想在执行过程中保存像PRD文件这些产物，可以在第3步前执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。 +# 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid-js +npm --version +sudo npm install -g @mermaid-js/mermaid-cli ``` 详细的安装请安装 [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) diff --git a/docs/README_JA.md b/docs/README_JA.md index dfb9c8372..0a65e83e0 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -60,17 +60,22 @@ ### インストールビデオガイド ### 伝統的なインストール ```bash -# ステップ 1: NPM がシステムにインストールされていることを確認してください。次に mermaid-js をインストールします。(お使いのコンピューターに npm がない場合は、Node.js 公式サイトで Node.js https://nodejs.org/ をインストールしてください。） -npm --version -sudo npm install -g @mermaid-js/mermaid-cli - -# ステップ 2: Python 3.9+ がシステムにインストールされていることを確認してください。これを確認するには: +# ステップ 1: Python 3.9+ がシステムにインストールされていることを確認してください。これを確認するには: python --version -# ステップ 3: リポジトリをローカルマシンにクローンし、インストールする。 +# ステップ 2: リポジトリをローカルマシンにクローンし、インストールする。 git clone https://github.com/geekan/MetaGPT.git cd MetaGPT pip install -e. + +# ステップ 3: startup.py を実行する +# config.yaml を key.yaml にコピーし、独自の OPENAI_API_KEY を設定します +python3 startup.py "Write a cli snake game" + +# ステップ 4 [オプション]: 実行中に PRD ファイルなどのアーティファクトを保存する場合は、ステップ 3 の前にこのステップを実行できます。デフォルトでは、フレームワークには互換性があり、この手順を実行しなくてもプロセス全体を完了できます。 +# NPM がシステムにインストールされていることを確認してください。次に mermaid-js をインストールします。(お使いのコンピューターに npm がない場合は、Node.js 公式サイトで Node.js https://nodejs.org/ をインストールしてください。） +npm --version +sudo npm install -g @mermaid-js/mermaid-cli ``` **注:** From fa6621a6996fdef77301f6ccbec08f4f95453514 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 20 Nov 2023 15:15:09 +0800 Subject: [PATCH 036/135] rm useless files --- docs/tutorial/faq.md | 6 ------ docs/tutorial/what_can_this_do.md | 0 2 files changed, 6 deletions(-) delete mode 100644 docs/tutorial/faq.md delete mode 100644 docs/tutorial/what_can_this_do.md diff --git a/docs/tutorial/faq.md b/docs/tutorial/faq.md deleted file mode 100644 index 0d2fa7ef0..000000000 --- a/docs/tutorial/faq.md +++ /dev/null @@ -1,6 +0,0 @@ -## FAQs - -### installation - - -### openai usage diff --git a/docs/tutorial/what_can_this_do.md b/docs/tutorial/what_can_this_do.md deleted file mode 100644 index e69de29bb..000000000 From c212c94da35ea80b63749bea536ea7003578a3d3 Mon Sep 17 00:00:00 2001 From: better629 Date: Mon, 20 Nov 2023 15:37:44 +0800 Subject: [PATCH 037/135] fix --- README.md | 2 +- docs/README_CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0606d2be6..a39b509df 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ # Step 3: run the startup.py # setup your OPENAI_API_KEY in key.yaml copy from config.yaml python3 startup.py "Write a cli snake game" -# Step 4 [Optional]: If you want to save the artifacts like PRD in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step. +# Step 4 [Optional]: If you want to save the artifacts like diagrams such as quadrant chart, system designs, sequence flow in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step. # If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) npm --version sudo npm install -g @mermaid-js/mermaid-cli diff --git a/docs/README_CN.md b/docs/README_CN.md index d38d6908b..50cf207b4 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -52,7 +52,7 @@ # 第 3 步：执行startup.py # 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY python3 startup.py "Write a cli snake game" -# 第 4 步【可选的】：如果你想在执行过程中保存像PRD文件这些产物，可以在第3步前执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。 +# 第 4 步【可选的】：如果你想在执行过程中保存像象限图、系统设计、序列流程等图表这些产物，可以在第3步前执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。 # 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid-js npm --version sudo npm install -g @mermaid-js/mermaid-cli From 0cf6ec1a93e40ad33ebb46b4060e10a312138253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:27:16 +0800 Subject: [PATCH 038/135] feat: +git repo --- metagpt/utils/git_repository.py | 110 +++++++++++++++++++++ tests/metagpt/utils/test_git_repository.py | 79 +++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 metagpt/utils/git_repository.py create mode 100644 tests/metagpt/utils/test_git_repository.py diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py new file mode 100644 index 000000000..fd9794a80 --- /dev/null +++ b/metagpt/utils/git_repository.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : git_repository.py +@Desc: Git repository management +""" +from __future__ import annotations + +import shutil +from enum import Enum +from pathlib import Path +from typing import Dict + +from git.repo import Repo +from git.repo.fun import is_git_dir + +from metagpt.const import WORKSPACE_ROOT + + +class ChangeType(Enum): + ADDED = "A" # File was added + COPIED = "C" # File was copied + DELETED = "D" # File was deleted + RENAMED = "R" # File was renamed + MODIFIED = "M" # File was modified + TYPE_CHANGED = "T" # Type of the file was changed + UNTRACTED = "U" # File is untracked (not added to version control) + + +class GitRepository: + def __init__(self, local_path=None, auto_init=True): + self._repository = None + if local_path: + self.open(local_path=local_path, auto_init=auto_init) + + def open(self, local_path: Path, auto_init=False): + if self.is_git_dir(local_path): + self._repository = Repo(local_path) + return + if not auto_init: + return + local_path.mkdir(parents=True, exist_ok=True) + return self._init(local_path) + + def _init(self, local_path: Path): + self._repository = Repo.init(path=local_path) + + def add_change(self, files: Dict): + if not self.is_valid or not files: + return + + for k, v in files.items(): + self._repository.index.remove(k) if v is ChangeType.DELETED else self._repository.index.add([k]) + + def commit(self, comments): + if self.is_valid: + self._repository.index.commit(comments) + + def delete_repository(self): + # Delete the repository directory + if self.is_valid: + shutil.rmtree(self._repository.working_dir) + + @property + def changed_files(self) -> Dict[str, str]: + files = {i: ChangeType.UNTRACTED for i in self._repository.untracked_files} + changed_files = {f.a_path: ChangeType(f.change_type) for f in self._repository.index.diff(None)} + files.update(changed_files) + return files + + @staticmethod + def is_git_dir(local_path): + git_dir = local_path / ".git" + if git_dir.exists() and is_git_dir(git_dir): + return True + return False + + @property + def is_valid(self): + return bool(self._repository) + + @property + def status(self) -> str: + if not self.is_valid: + return "" + return self._repository.git.status() + + @property + def workdir(self) -> Path | None: + if not self.is_valid: + return None + return Path(self._repository.working_dir) + + +if __name__ == "__main__": + path = WORKSPACE_ROOT / "git" + path.mkdir(exist_ok=True, parents=True) + + repo = GitRepository() + repo.open(path, auto_init=True) + + changes = repo.changed_files + print(changes) + repo.add_change(changes) + print(repo.status) + repo.commit("test") + print(repo.status) + repo.delete_repository() diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py new file mode 100644 index 000000000..2e15f44f9 --- /dev/null +++ b/tests/metagpt/utils/test_git_repository.py @@ -0,0 +1,79 @@ +import shutil +from pathlib import Path + +import aiofiles +import pytest + +from metagpt.utils.git_repository import GitRepository + + +async def mock_file(filename, content=""): + async with aiofiles.open(str(filename), mode="w") as file: + await file.write(content) + + +@pytest.mark.asyncio +async def test_git(): + local_path = Path(__file__).parent / "git" + if local_path.exists(): + shutil.rmtree(local_path) + assert not local_path.exists() + repo = GitRepository(local_path=local_path, auto_init=True) + assert local_path.exists() + assert local_path == repo.workdir + assert not repo.changed_files + + await mock_file(local_path / "a.txt") + await mock_file(local_path / "b.txt") + subdir = local_path / "subdir" + subdir.mkdir(parents=True, exist_ok=True) + await mock_file(subdir / "c.txt") + + assert len(repo.changed_files) == 3 + repo.add_change(repo.changed_files) + repo.commit("commit1") + assert not repo.changed_files + + await mock_file(local_path / "a.txt", "tests") + await mock_file(subdir / "d.txt") + rmfile = local_path / "b.txt" + rmfile.unlink() + assert repo.status + + assert len(repo.changed_files) == 3 + repo.add_change(repo.changed_files) + repo.commit("commit2") + assert not repo.changed_files + + assert repo.status + + repo.delete_repository() + assert not local_path.exists() + + +@pytest.mark.asyncio +async def test_git1(): + local_path = Path(__file__).parent / "git1" + if local_path.exists(): + shutil.rmtree(local_path) + assert not local_path.exists() + repo = GitRepository(local_path=local_path, auto_init=True) + assert local_path.exists() + assert local_path == repo.workdir + assert not repo.changed_files + + await mock_file(local_path / "a.txt") + await mock_file(local_path / "b.txt") + subdir = local_path / "subdir" + subdir.mkdir(parents=True, exist_ok=True) + await mock_file(subdir / "c.txt") + + repo1 = GitRepository(local_path=local_path, auto_init=False) + assert repo1.changed_files + + repo1.delete_repository() + assert not local_path.exists() + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 9c5f7c76719e07845da74c7ef915388b44722433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:33:46 +0800 Subject: [PATCH 039/135] feat: +annotation --- metagpt/utils/git_repository.py | 51 +++++++++++++++++++++- tests/metagpt/utils/test_git_repository.py | 25 ++++------- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index fd9794a80..c5b510612 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -30,12 +30,31 @@ class ChangeType(Enum): class GitRepository: + """A class representing a Git repository. + + :param local_path: The local path to the Git repository. + :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. + + Attributes: + _repository (Repo): The GitPython `Repo` object representing the Git repository. + """ + def __init__(self, local_path=None, auto_init=True): + """Initialize a GitRepository instance. + + :param local_path: The local path to the Git repository. + :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. + """ self._repository = None if local_path: self.open(local_path=local_path, auto_init=auto_init) def open(self, local_path: Path, auto_init=False): + """Open an existing Git repository or initialize a new one if auto_init is True. + + :param local_path: The local path to the Git repository. + :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. + """ if self.is_git_dir(local_path): self._repository = Repo(local_path) return @@ -45,9 +64,17 @@ class GitRepository: return self._init(local_path) def _init(self, local_path: Path): + """Initialize a new Git repository at the specified path. + + :param local_path: The local path where the new Git repository will be initialized. + """ self._repository = Repo.init(path=local_path) def add_change(self, files: Dict): + """Add or remove files from the staging area based on the provided changes. + + :param files: A dictionary where keys are file paths and values are instances of ChangeType. + """ if not self.is_valid or not files: return @@ -55,16 +82,24 @@ class GitRepository: self._repository.index.remove(k) if v is ChangeType.DELETED else self._repository.index.add([k]) def commit(self, comments): + """Commit the staged changes with the given comments. + + :param comments: Comments for the commit. + """ if self.is_valid: self._repository.index.commit(comments) def delete_repository(self): - # Delete the repository directory + """Delete the entire repository directory.""" if self.is_valid: shutil.rmtree(self._repository.working_dir) @property def changed_files(self) -> Dict[str, str]: + """Return a dictionary of changed files and their change types. + + :return: A dictionary where keys are file paths and values are change types. + """ files = {i: ChangeType.UNTRACTED for i in self._repository.untracked_files} changed_files = {f.a_path: ChangeType(f.change_type) for f in self._repository.index.diff(None)} files.update(changed_files) @@ -72,6 +107,11 @@ class GitRepository: @staticmethod def is_git_dir(local_path): + """Check if the specified directory is a Git repository. + + :param local_path: The local path to check. + :return: True if the directory is a Git repository, False otherwise. + """ git_dir = local_path / ".git" if git_dir.exists() and is_git_dir(git_dir): return True @@ -79,16 +119,25 @@ class GitRepository: @property def is_valid(self): + """Check if the Git repository is valid (exists and is initialized). + + :return: True if the repository is valid, False otherwise. + """ return bool(self._repository) @property def status(self) -> str: + """Return the Git repository's status as a string.""" if not self.is_valid: return "" return self._repository.git.status() @property def workdir(self) -> Path | None: + """Return the path to the working directory of the Git repository. + + :return: The path to the working directory or None if the repository is not valid. + """ if not self.is_valid: return None return Path(self._repository.working_dir) diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index 2e15f44f9..fa329a2ec 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -12,9 +12,7 @@ async def mock_file(filename, content=""): await file.write(content) -@pytest.mark.asyncio -async def test_git(): - local_path = Path(__file__).parent / "git" +async def mock_repo(local_path) -> (GitRepository, Path): if local_path.exists(): shutil.rmtree(local_path) assert not local_path.exists() @@ -28,6 +26,13 @@ async def test_git(): subdir = local_path / "subdir" subdir.mkdir(parents=True, exist_ok=True) await mock_file(subdir / "c.txt") + return repo, subdir + + +@pytest.mark.asyncio +async def test_git(): + local_path = Path(__file__).parent / "git" + repo, subdir = await mock_repo(local_path) assert len(repo.changed_files) == 3 repo.add_change(repo.changed_files) @@ -54,19 +59,7 @@ async def test_git(): @pytest.mark.asyncio async def test_git1(): local_path = Path(__file__).parent / "git1" - if local_path.exists(): - shutil.rmtree(local_path) - assert not local_path.exists() - repo = GitRepository(local_path=local_path, auto_init=True) - assert local_path.exists() - assert local_path == repo.workdir - assert not repo.changed_files - - await mock_file(local_path / "a.txt") - await mock_file(local_path / "b.txt") - subdir = local_path / "subdir" - subdir.mkdir(parents=True, exist_ok=True) - await mock_file(subdir / "c.txt") + await mock_repo(local_path) repo1 = GitRepository(local_path=local_path, auto_init=False) assert repo1.changed_files From f1fb3b3bece668590557455ae51ecf9b8f306109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:35:16 +0800 Subject: [PATCH 040/135] feat: +annotation --- tests/metagpt/utils/test_git_repository.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index fa329a2ec..0d1e3b791 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : test_git_repository.py +@Desc: Unit tests for git_repository.py +""" + import shutil from pathlib import Path From 363be23045e552d324e2946d16bc0eb29d5302f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:44:06 +0800 Subject: [PATCH 041/135] feat: +annotation --- metagpt/utils/file_repository.py | 0 metagpt/utils/git_repository.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 metagpt/utils/file_repository.py diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py new file mode 100644 index 000000000..e69de29bb diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index c5b510612..1732d6a91 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -4,7 +4,7 @@ @Time : 2023/11/20 @Author : mashenquan @File : git_repository.py -@Desc: Git repository management +@Desc: Git repository management. RFC 135 2.2.3.3. """ from __future__ import annotations From af716c6c305254cfb48fda5a865616126931edd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 16:47:12 +0800 Subject: [PATCH 042/135] feat: +annotation --- metagpt/utils/file_repository.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 metagpt/utils/file_repository.py diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py deleted file mode 100644 index e69de29bb..000000000 From 990d79179f48ffc5afce0276dec1cfeb2db4ef9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 17:33:24 +0800 Subject: [PATCH 043/135] feat: archive --- metagpt/utils/file_repository.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 metagpt/utils/file_repository.py diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py new file mode 100644 index 000000000..e69de29bb From 9f7da1c7688f48d9a7ac2cf38c7f81cca35f7ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 17:34:44 +0800 Subject: [PATCH 044/135] feat: archive --- metagpt/utils/file_repository.py | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index e69de29bb..af787c70a 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : git_repository.py +@Desc: File repository management. RFC 135 2.2.3.2, 2.2.3.4 and 2.2.3.13. +""" +import json +from pathlib import Path +from typing import Dict, List + +import aiofiles + +from metagpt.utils.git_repository import GitRepository + + +class FileRepository: + def __init__(self, git_repo: GitRepository, relative_path: Path = "."): + self._relative_path = relative_path # Relative path based on the Git repository. + self._git_repo = git_repo + self._dependencies: Dict[str, List[str]] = {} + + async def save(self, filename: Path, content, dependencies: List[str] = None): + path_name = self.workdir / filename + with aiofiles.open(str(path_name), mode="w") as writer: + await writer.write(content) + if dependencies is not None: + await self.update_dependency(filename, dependencies) + + async def update_dependency(self, filename, dependencies: List[str]): + self._dependencies[str(filename)] = dependencies + + async def save_dependency(self): + filename = ".dependencies.json" + path_name = self.workdir / filename + data = json.dumps(self._dependencies) + with aiofiles.open(str(path_name), mode="w") as writer: + await writer.write(data) + + @property + def workdir(self): + return self._git_repo.workdir / self._relative_path From 913cfaebabc22d1130bb9cff9b8a4713b2cd72cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 17:41:16 +0800 Subject: [PATCH 045/135] feat: archive --- metagpt/utils/git_repository.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 1732d6a91..6e624c8b5 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -142,6 +142,14 @@ class GitRepository: return None return Path(self._repository.working_dir) + def archive(self, comments="Archive"): + """Archive the current state of the Git repository. + + :param comments: Comments for the archive commit. + """ + self.add_change(self.changed_files) + self.commit(comments) + if __name__ == "__main__": path = WORKSPACE_ROOT / "git" From 29003a9beb0f1ede36c1139ee8bb3815e0fdad49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 19:36:54 +0800 Subject: [PATCH 046/135] feat: +file repository --- metagpt/utils/file_repository.py | 71 +++++++++++++++++++-- metagpt/utils/git_repository.py | 9 +++ tests/metagpt/utils/test_file_repository.py | 49 ++++++++++++++ 3 files changed, 122 insertions(+), 7 deletions(-) create mode 100644 tests/metagpt/utils/test_file_repository.py diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index af787c70a..d8637fe3f 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -6,38 +6,95 @@ @File : git_repository.py @Desc: File repository management. RFC 135 2.2.3.2, 2.2.3.4 and 2.2.3.13. """ +from __future__ import annotations + import json from pathlib import Path from typing import Dict, List import aiofiles -from metagpt.utils.git_repository import GitRepository +from metagpt.logs import logger class FileRepository: - def __init__(self, git_repo: GitRepository, relative_path: Path = "."): + def __init__(self, git_repo, relative_path: Path = Path(".")): self._relative_path = relative_path # Relative path based on the Git repository. self._git_repo = git_repo self._dependencies: Dict[str, List[str]] = {} - async def save(self, filename: Path, content, dependencies: List[str] = None): + # Initializing + self.workdir.mkdir(parents=True, exist_ok=True) + if self.dependency_path_name.exists(): + try: + with open(str(self.dependency_path_name), mode="r") as reader: + self._dependencies = json.load(reader) + except Exception as e: + logger.error(f"Failed to load {str(self.dependency_path_name)}, error:{e}") + + async def save(self, filename: Path | str, content, dependencies: List[str] = None): path_name = self.workdir / filename - with aiofiles.open(str(path_name), mode="w") as writer: + path_name.parent.mkdir(parents=True, exist_ok=True) + async with aiofiles.open(str(path_name), mode="w") as writer: await writer.write(content) if dependencies is not None: await self.update_dependency(filename, dependencies) + async def get(self, filename: Path | str): + path_name = self.workdir / filename + async with aiofiles.open(str(path_name), mode="r") as reader: + return await reader.read() + + def get_dependency(self, filename: Path | str) -> List: + key = str(filename) + return self._dependencies.get(key, []) + + def get_changed_dependency(self, filename: Path | str) -> List: + dependencies = self.get_dependency(filename=filename) + changed_files = self.changed_files + changed_dependent_files = [] + for df in dependencies: + if df in changed_files.keys(): + changed_dependent_files.append(df) + return changed_dependent_files + async def update_dependency(self, filename, dependencies: List[str]): self._dependencies[str(filename)] = dependencies async def save_dependency(self): - filename = ".dependencies.json" - path_name = self.workdir / filename data = json.dumps(self._dependencies) - with aiofiles.open(str(path_name), mode="w") as writer: + with aiofiles.open(str(self.dependency_path_name), mode="w") as writer: await writer.write(data) @property def workdir(self): return self._git_repo.workdir / self._relative_path + + @property + def dependency_path_name(self): + filename = ".dependencies.json" + path_name = self.workdir / filename + return path_name + + @property + def changed_files(self) -> Dict[str, str]: + files = self._git_repo.changed_files + relative_files = {} + for p, ct in files.items(): + try: + rf = Path(p).relative_to(self._relative_path) + except ValueError: + continue + relative_files[str(rf)] = ct + return relative_files + + def get_change_dir_files(self, dir: Path | str) -> List: + changed_files = self.changed_files + children = [] + for f in changed_files: + try: + Path(f).relative_to(Path(dir)) + except ValueError: + continue + children.append(str(f)) + return children diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 6e624c8b5..6ae6a7900 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -17,6 +17,7 @@ from git.repo import Repo from git.repo.fun import is_git_dir from metagpt.const import WORKSPACE_ROOT +from metagpt.utils.file_repository import FileRepository class ChangeType(Enum): @@ -150,6 +151,14 @@ class GitRepository: self.add_change(self.changed_files) self.commit(comments) + def new_file_repository(self, relative_path: Path | str) -> FileRepository: + """Create a new instance of FileRepository associated with this Git repository. + + :param relative_path: The relative path to the file repository within the Git repository. + :return: A new instance of FileRepository. + """ + return FileRepository(git_repo=self, relative_path=Path(relative_path)) + if __name__ == "__main__": path = WORKSPACE_ROOT / "git" diff --git a/tests/metagpt/utils/test_file_repository.py b/tests/metagpt/utils/test_file_repository.py new file mode 100644 index 000000000..ac36f2320 --- /dev/null +++ b/tests/metagpt/utils/test_file_repository.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : test_file_repository.py +@Desc: Unit tests for file_repository.py +""" +import shutil +from pathlib import Path + +import pytest + +from metagpt.utils.git_repository import ChangeType, GitRepository +from tests.metagpt.utils.test_git_repository import mock_file + + +@pytest.mark.asyncio +async def test_file_repo(): + local_path = Path(__file__).parent / "file_repo_git" + if local_path.exists(): + shutil.rmtree(local_path) + + git_repo = GitRepository(local_path=local_path, auto_init=True) + assert not git_repo.changed_files + + await mock_file(local_path / "g.txt", "") + + file_repo_path = "file_repo1" + full_path = local_path / file_repo_path + assert not full_path.exists() + file_repo = git_repo.new_file_repository(file_repo_path) + assert file_repo.workdir == full_path + assert file_repo.workdir.exists() + await file_repo.save("a.txt", "AAA") + await file_repo.save("b.txt", "BBB", ["a.txt"]) + assert "AAA" == await file_repo.get("a.txt") + assert "BBB" == await file_repo.get("b.txt") + assert ["a.txt"] == file_repo.get_dependency("b.txt") + assert {"a.txt": ChangeType.UNTRACTED, "b.txt": ChangeType.UNTRACTED} == file_repo.changed_files + assert ["a.txt"] == file_repo.get_changed_dependency("b.txt") + await file_repo.save("d/e.txt", "EEE") + assert ["d/e.txt"] == file_repo.get_change_dir_files("d") + + git_repo.delete_repository() + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) From 85e3620638348826f32f47065292c91e9e845193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 20 Nov 2023 19:40:15 +0800 Subject: [PATCH 047/135] feat: +file repository --- metagpt/utils/file_repository.py | 51 +++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index d8637fe3f..f4c36b5b7 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -19,7 +19,12 @@ from metagpt.logs import logger class FileRepository: def __init__(self, git_repo, relative_path: Path = Path(".")): - self._relative_path = relative_path # Relative path based on the Git repository. + """Initialize a FileRepository instance. + + :param git_repo: The associated GitRepository instance. + :param relative_path: The relative path within the Git repository. + """ + self._relative_path = relative_path self._git_repo = git_repo self._dependencies: Dict[str, List[str]] = {} @@ -33,6 +38,12 @@ class FileRepository: logger.error(f"Failed to load {str(self.dependency_path_name)}, error:{e}") async def save(self, filename: Path | str, content, dependencies: List[str] = None): + """Save content to a file and update its dependencies. + + :param filename: The filename or path within the repository. + :param content: The content to be saved. + :param dependencies: List of dependency filenames or paths. + """ path_name = self.workdir / filename path_name.parent.mkdir(parents=True, exist_ok=True) async with aiofiles.open(str(path_name), mode="w") as writer: @@ -41,15 +52,30 @@ class FileRepository: await self.update_dependency(filename, dependencies) async def get(self, filename: Path | str): + """Read the content of a file. + + :param filename: The filename or path within the repository. + :return: The content of the file. + """ path_name = self.workdir / filename async with aiofiles.open(str(path_name), mode="r") as reader: return await reader.read() def get_dependency(self, filename: Path | str) -> List: + """Get the dependencies of a file. + + :param filename: The filename or path within the repository. + :return: List of dependency filenames or paths. + """ key = str(filename) return self._dependencies.get(key, []) def get_changed_dependency(self, filename: Path | str) -> List: + """Get the dependencies of a file that have changed. + + :param filename: The filename or path within the repository. + :return: List of changed dependency filenames or paths. + """ dependencies = self.get_dependency(filename=filename) changed_files = self.changed_files changed_dependent_files = [] @@ -59,25 +85,43 @@ class FileRepository: return changed_dependent_files async def update_dependency(self, filename, dependencies: List[str]): + """Update the dependencies of a file. + + :param filename: The filename or path within the repository. + :param dependencies: List of dependency filenames or paths. + """ self._dependencies[str(filename)] = dependencies async def save_dependency(self): + """Save the dependencies to a file.""" data = json.dumps(self._dependencies) with aiofiles.open(str(self.dependency_path_name), mode="w") as writer: await writer.write(data) @property def workdir(self): + """Return the absolute path to the working directory of the FileRepository. + + :return: The absolute path to the working directory. + """ return self._git_repo.workdir / self._relative_path @property def dependency_path_name(self): + """Return the absolute path to the dependency file. + + :return: The absolute path to the dependency file. + """ filename = ".dependencies.json" path_name = self.workdir / filename return path_name @property def changed_files(self) -> Dict[str, str]: + """Return a dictionary of changed files and their change types. + + :return: A dictionary where keys are file paths and values are change types. + """ files = self._git_repo.changed_files relative_files = {} for p, ct in files.items(): @@ -89,6 +133,11 @@ class FileRepository: return relative_files def get_change_dir_files(self, dir: Path | str) -> List: + """Get the files in a directory that have changed. + + :param dir: The directory path within the repository. + :return: List of changed filenames or paths within the directory. + """ changed_files = self.changed_files children = [] for f in changed_files: From daf29c41afba445bd507d75e6549e0f8fb14ecc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Tue, 21 Nov 2023 11:17:33 +0800 Subject: [PATCH 048/135] chore: change name. --- metagpt/provider/constant.py | 4 ++-- metagpt/provider/openai_api.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/provider/constant.py b/metagpt/provider/constant.py index f9b84fc60..f228e7c30 100644 --- a/metagpt/provider/constant.py +++ b/metagpt/provider/constant.py @@ -1,6 +1,6 @@ # function in tools, https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools # Reference: https://github.com/KillianLucas/open-interpreter/blob/v0.1.14/interpreter/llm/setup_openai_coding_llm.py -general_function_schema = { +GENRAL_FUNCTION_SCHEMA_4_ASK_CODE = { "name": "execute", "description": "Executes code on the user's machine, **in the users local environment**, and returns the output", "parameters": { @@ -27,4 +27,4 @@ general_function_schema = { # tool_choice value for general_function_schema # https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice -general_tool_choice = {"type": "function", "function": {"name": "execute"}} +GENRAL_TOOL_CHOICE_4_ASK_CODE = {"type": "function", "function": {"name": "execute"}} diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 8f8d45d73..f950c0d67 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -21,7 +21,7 @@ from tenacity import ( from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.provider.constant import general_function_schema, general_tool_choice +from metagpt.provider.constant import GENRAL_FUNCTION_SCHEMA_4_ASK_CODE, GENRAL_TOOL_CHOICE_4_ASK_CODE from metagpt.schema import Message from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( @@ -245,8 +245,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def _func_configs(self, messages: list[dict], **kwargs) -> dict: if "tools" not in kwargs: configs = { - "tools": [{"type": "function", "function": general_function_schema}], - "tool_choice": general_tool_choice, + "tools": [{"type": "function", "function": GENRAL_FUNCTION_SCHEMA_4_ASK_CODE}], + "tool_choice": GENRAL_TOOL_CHOICE_4_ASK_CODE, } kwargs.update(configs) From fcc9bd6063b77c758fd10c9d78affde7a93236ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Tue, 21 Nov 2023 11:17:56 +0800 Subject: [PATCH 049/135] chore: openai -> openai>=0.28.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 093298775..ff483e97f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ langchain==0.0.231 loguru==0.6.0 meilisearch==0.21.0 numpy==1.24.3 -openai +openai>=0.28.0 openpyxl beautifulsoup4==4.12.2 pandas==2.0.3 From 66f647276e8775478321cd19bddb5f7cce9000d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Tue, 21 Nov 2023 11:19:43 +0800 Subject: [PATCH 050/135] Revert "chore: change name." This reverts commit daf29c41afba445bd507d75e6549e0f8fb14ecc5. --- metagpt/provider/constant.py | 4 ++-- metagpt/provider/openai_api.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/provider/constant.py b/metagpt/provider/constant.py index f228e7c30..f9b84fc60 100644 --- a/metagpt/provider/constant.py +++ b/metagpt/provider/constant.py @@ -1,6 +1,6 @@ # function in tools, https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools # Reference: https://github.com/KillianLucas/open-interpreter/blob/v0.1.14/interpreter/llm/setup_openai_coding_llm.py -GENRAL_FUNCTION_SCHEMA_4_ASK_CODE = { +general_function_schema = { "name": "execute", "description": "Executes code on the user's machine, **in the users local environment**, and returns the output", "parameters": { @@ -27,4 +27,4 @@ GENRAL_FUNCTION_SCHEMA_4_ASK_CODE = { # tool_choice value for general_function_schema # https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice -GENRAL_TOOL_CHOICE_4_ASK_CODE = {"type": "function", "function": {"name": "execute"}} +general_tool_choice = {"type": "function", "function": {"name": "execute"}} diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index f950c0d67..8f8d45d73 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -21,7 +21,7 @@ from tenacity import ( from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.provider.constant import GENRAL_FUNCTION_SCHEMA_4_ASK_CODE, GENRAL_TOOL_CHOICE_4_ASK_CODE +from metagpt.provider.constant import general_function_schema, general_tool_choice from metagpt.schema import Message from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( @@ -245,8 +245,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def _func_configs(self, messages: list[dict], **kwargs) -> dict: if "tools" not in kwargs: configs = { - "tools": [{"type": "function", "function": GENRAL_FUNCTION_SCHEMA_4_ASK_CODE}], - "tool_choice": GENRAL_TOOL_CHOICE_4_ASK_CODE, + "tools": [{"type": "function", "function": general_function_schema}], + "tool_choice": general_tool_choice, } kwargs.update(configs) From 4418fe8283635b5bdf64489b916c073d97695f80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Tue, 21 Nov 2023 11:40:15 +0800 Subject: [PATCH 051/135] feat: ask_code message support type list[Message]. --- metagpt/provider/openai_api.py | 4 ++-- tests/metagpt/provider/test_openai.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 8f8d45d73..11f429601 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -262,10 +262,10 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self._update_costs(rsp.get("usage")) return rsp - def _process_message(self, messages: Union[str, Message, list[dict]]) -> list[dict]: + def _process_message(self, messages: Union[str, Message, list[dict], list[Message]]) -> list[dict]: """convert messages to list[dict].""" if isinstance(messages, list): - return messages + return [msg if isinstance(msg, dict) else msg.to_dict() for msg in messages] if isinstance(messages, Message): messages = [messages.to_dict()] diff --git a/tests/metagpt/provider/test_openai.py b/tests/metagpt/provider/test_openai.py index d6d9f4f9d..93ac6623c 100644 --- a/tests/metagpt/provider/test_openai.py +++ b/tests/metagpt/provider/test_openai.py @@ -59,3 +59,12 @@ def test_ask_code_Message(): assert "language" in rsp assert "code" in rsp assert len(rsp["code"]) > 0 + + +def test_ask_code_list_Message(): + llm = OpenAIGPTAPI() + msg = [UserMessage("a=[1,2,5,10,-10]"), UserMessage("写出求a中最大值的代码python")] + rsp = llm.ask_code(msg) # -> {'language': 'python', 'code': 'max_value = max(a)\nmax_value'} + assert "language" in rsp + assert "code" in rsp + assert len(rsp["code"]) > 0 From 657571a192bab2dd092ac4bf5860411489dc9e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Tue, 21 Nov 2023 12:01:28 +0800 Subject: [PATCH 052/135] feat: ask_code message support type list[str]. --- metagpt/provider/openai_api.py | 3 ++- tests/metagpt/provider/test_openai.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 11f429601..ef9b790ce 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -262,9 +262,10 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self._update_costs(rsp.get("usage")) return rsp - def _process_message(self, messages: Union[str, Message, list[dict], list[Message]]) -> list[dict]: + def _process_message(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: """convert messages to list[dict].""" if isinstance(messages, list): + messages = [Message(msg) if isinstance(msg, str) else msg for msg in messages] return [msg if isinstance(msg, dict) else msg.to_dict() for msg in messages] if isinstance(messages, Message): diff --git a/tests/metagpt/provider/test_openai.py b/tests/metagpt/provider/test_openai.py index 93ac6623c..2b0af37b5 100644 --- a/tests/metagpt/provider/test_openai.py +++ b/tests/metagpt/provider/test_openai.py @@ -68,3 +68,13 @@ def test_ask_code_list_Message(): assert "language" in rsp assert "code" in rsp assert len(rsp["code"]) > 0 + + +def test_ask_code_list_str(): + llm = OpenAIGPTAPI() + msg = ["a=[1,2,5,10,-10]", "写出求a中最大值的代码python"] + rsp = llm.ask_code(msg) # -> {'language': 'python', 'code': 'max_value = max(a)\nmax_value'} + print(rsp) + assert "language" in rsp + assert "code" in rsp + assert len(rsp["code"]) > 0 From d9a2626fde3c7e43e7e118e3ee740a0ad4b9fcf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 21 Nov 2023 13:39:10 +0800 Subject: [PATCH 053/135] feat: +PrepareDocuments --- metagpt/actions/prepare_documents.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 metagpt/actions/prepare_documents.py diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py new file mode 100644 index 000000000..7cf05c5d1 --- /dev/null +++ b/metagpt/actions/prepare_documents.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/20 +@Author : mashenquan +@File : git_repository.py +@Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. + RFC 135 2.2.3.5.1. +""" +from metagpt.actions import Action + + +class PrepareDocuments(Action): + def __init__(self, name="", context=None, llm=None): + pass From 9d1d8a9fe4c7946c3c36995d6584491dfd17f97c Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 21 Nov 2023 14:10:04 +0800 Subject: [PATCH 054/135] fix --- metagpt/provider/zhipuai_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 2ad1944c2..064ec35ba 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -20,7 +20,6 @@ from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.openai_api import CostManager, log_and_reraise from metagpt.provider.zhipuai.zhipu_model_api import ZhiPuModelAPI -from metagpt.utils.ahttp_client import astream class ZhiPuEvent(Enum): From ded2044be7eb244ad0549873b5041945a8da674d Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 21 Nov 2023 14:50:31 +0800 Subject: [PATCH 055/135] rm useless func --- metagpt/provider/general_api_requestor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/general_api_requestor.py b/metagpt/provider/general_api_requestor.py index 169b7c146..150f2f1e0 100644 --- a/metagpt/provider/general_api_requestor.py +++ b/metagpt/provider/general_api_requestor.py @@ -6,7 +6,7 @@ from typing import AsyncGenerator, Tuple, Union, Optional, Literal import aiohttp import asyncio -from openai.api_requestor import APIRequestor, aiohttp_session +from openai.api_requestor import APIRequestor from metagpt.logs import logger From 27c731d11a94901d4b51dbfe57042ee1a9681b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 21 Nov 2023 15:05:23 +0800 Subject: [PATCH 056/135] feat: archive --- metagpt/actions/prepare_documents.py | 12 +++++++++++- metagpt/environment.py | 9 ++++++++- metagpt/roles/product_manager.py | 11 ++++++++++- metagpt/roles/role.py | 4 ++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 7cf05c5d1..b0185996b 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -12,4 +12,14 @@ from metagpt.actions import Action class PrepareDocuments(Action): def __init__(self, name="", context=None, llm=None): - pass + super().__init__(name, context, llm) + + async def run(self, with_message, **kwargs): + parent = self.context.get("parent") + if not parent: + raise ValueError("Invalid owner") + env = parent.get_env() + if env.git_repository: + return + env.git_repository = GitRepository() + env.git_repository.open(WORKS) diff --git a/metagpt/environment.py b/metagpt/environment.py index b3c296dac..df93a818b 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -12,7 +12,7 @@ functionality is to be consolidated into the `Environment` class. """ import asyncio -from typing import Iterable, Set +from typing import Iterable, Optional, Set from pydantic import BaseModel, Field @@ -20,6 +20,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import is_subscribed +from metagpt.utils.git_repository import GitRepository class Environment(BaseModel): @@ -31,6 +32,7 @@ class Environment(BaseModel): roles: dict[str, Role] = Field(default_factory=dict) consumers: dict[Role, Set] = Field(default_factory=dict) history: str = Field(default="") # For debug + git_repository: Optional[GitRepository] = None class Config: arbitrary_types_allowed = True @@ -111,3 +113,8 @@ class Environment(BaseModel): def set_subscription(self, obj, tags): """Set the labels for message to be consumed by the object""" self.consumers[obj] = tags + + def dict(self, *args, **kwargs): + """Generate a dictionary representation of the model, optionally specifying which fields to include or + exclude.""" + return super(Environment, self).dict(exclude={"git_repository"}) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index a58ea5385..c10aba6d1 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -6,6 +6,7 @@ @File : product_manager.py """ from metagpt.actions import BossRequirement, WritePRD +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.roles import Role @@ -37,5 +38,13 @@ class ProductManager(Role): constraints (str): Constraints or limitations for the product manager. """ super().__init__(name, profile, goal, constraints) - self._init_actions([WritePRD]) + self._init_actions([PrepareDocuments(context={"parent": self}), WritePRD]) self._watch([BossRequirement]) + + async def _think(self) -> None: + """Decide what to do""" + if self._rc.env.git_repository: + self._set_state(1) + else: + self._set_state(0) + return self._rc.todo diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 2e3bcbbd5..d1e65a4e0 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -164,6 +164,10 @@ class Role: if env: env.set_subscription(self, self._subscription) + def get_env(self): + """Return the environment in which the role works.""" + return self._rc.env + @property def profile(self): """Get the role description (position)""" From 45655a2a28efc07fcaacd7d91106eaa1f2d12984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Tue, 21 Nov 2023 16:34:12 +0800 Subject: [PATCH 057/135] chore: add function signature to get_choice_function and get_choice_function_arguments. --- metagpt/provider/base_gpt_api.py | 39 +++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 2cfc3fa1f..f23ef871c 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -111,16 +111,43 @@ class BaseGPTAPI(BaseChatbot): return rsp.get("choices")[0]["message"]["content"] def get_choice_function(self, rsp: dict) -> dict: - """Required to provide the first function of choice. for example: - "function": { - "name": "execute", - "arguments": "{\n \"language\": \"python\",\n \"code\": \"print('Hello, World!')\"\n}" - } + """Required to provide the first function of choice + :param dict rsp: OpenAI chat.comletion respond JSON, Note "message" must include "tool_calls", + and "tool_calls" must include "function", for example: + {... + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_Y5r6Ddr2Qc2ZrqgfwzPX5l72", + "type": "function", + "function": { + "name": "execute", + "arguments": "{\n \"language\": \"python\",\n \"code\": \"print('Hello, World!')\"\n}" + } + } + ] + }, + "finish_reason": "stop" + } + ], + ...} + :return dict: return first function of choice, for exmaple, + {'name': 'execute', 'arguments': '{\n "language": "python",\n "code": "print(\'Hello, World!\')"\n}'} """ return rsp.get("choices")[0]["message"]["tool_calls"][0]["function"].to_dict() def get_choice_function_arguments(self, rsp: dict) -> dict: - """Required to provide the first function arguments of choice.""" + """Required to provide the first function arguments of choice. + + :param dict rsp: same as in self.get_choice_function(rsp) + :return dict: return the first function arguments of choice, for example, + {'language': 'python', 'code': "print('Hello, World!')"} + """ return json.loads(self.get_choice_function(rsp)["arguments"]) def messages_to_prompt(self, messages: list[dict]): From bec4bc0e8386a9866904ab92cc55df1d155549df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Tue, 21 Nov 2023 17:01:03 +0800 Subject: [PATCH 058/135] chore: add comments for `kwargs`. --- metagpt/provider/openai_api.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index ef9b790ce..af7810b62 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -243,6 +243,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return self.get_choice_text(rsp) def _func_configs(self, messages: list[dict], **kwargs) -> dict: + """ + Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create + """ if "tools" not in kwargs: configs = { "tools": [{"type": "function", "function": general_function_schema}], @@ -280,7 +283,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def ask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: """Use function of tools to ask a code. - https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools + + Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create Examples: @@ -297,7 +301,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: """Use function of tools to ask a code. - https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools + + Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create Examples: From 7a5c66b01edcd01ed26426df3ce6b7b2c0cfd733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E6=A3=92=E6=A3=92?= Date: Tue, 21 Nov 2023 17:14:39 +0800 Subject: [PATCH 059/135] chore: general_function_schema, general_tool_choice -> GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE --- metagpt/provider/constant.py | 4 ++-- metagpt/provider/openai_api.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/provider/constant.py b/metagpt/provider/constant.py index f9b84fc60..db67847a8 100644 --- a/metagpt/provider/constant.py +++ b/metagpt/provider/constant.py @@ -1,6 +1,6 @@ # function in tools, https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools # Reference: https://github.com/KillianLucas/open-interpreter/blob/v0.1.14/interpreter/llm/setup_openai_coding_llm.py -general_function_schema = { +GENERAL_FUNCTION_SCHEMA = { "name": "execute", "description": "Executes code on the user's machine, **in the users local environment**, and returns the output", "parameters": { @@ -27,4 +27,4 @@ general_function_schema = { # tool_choice value for general_function_schema # https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice -general_tool_choice = {"type": "function", "function": {"name": "execute"}} +GENERAL_TOOL_CHOICE = {"type": "function", "function": {"name": "execute"}} diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index af7810b62..34e5693f8 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -21,7 +21,7 @@ from tenacity import ( from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.provider.constant import general_function_schema, general_tool_choice +from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE from metagpt.schema import Message from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( @@ -248,8 +248,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ if "tools" not in kwargs: configs = { - "tools": [{"type": "function", "function": general_function_schema}], - "tool_choice": general_tool_choice, + "tools": [{"type": "function", "function": GENERAL_FUNCTION_SCHEMA}], + "tool_choice": GENERAL_TOOL_CHOICE, } kwargs.update(configs) From dfe91685098312f3c8eb729d6ebb0c1a411e23b6 Mon Sep 17 00:00:00 2001 From: Sirui Hong <34952977+stellaHSR@users.noreply.github.com> Date: Tue, 21 Nov 2023 17:35:33 +0800 Subject: [PATCH 060/135] Update llm.py rm 'sk-' verification --- metagpt/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 51ade1f7e..b653df946 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -18,7 +18,7 @@ from metagpt.provider.human_provider import HumanProvider def LLM() -> "BaseGPTAPI": """ initialize different LLM instance according to the key field existence""" # TODO a little trick, can use registry to initialize LLM instance further - if CONFIG.openai_api_key and CONFIG.openai_api_key.startswith("sk-"): + if CONFIG.openai_api_key: llm = OpenAIGPTAPI() elif CONFIG.claude_api_key: llm = Claude() From 322aef6f1d24e86e40718b702a4cb11660061822 Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 21 Nov 2023 19:24:57 +0800 Subject: [PATCH 061/135] fix zhipuapi's key to init openai api_key when using openai sdk --- metagpt/llm.py | 2 +- metagpt/provider/zhipuai_api.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index b653df946..4edcd7a83 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -24,7 +24,7 @@ def LLM() -> "BaseGPTAPI": llm = Claude() elif CONFIG.spark_api_key: llm = SparkAPI() - elif CONFIG.zhipuai_api_key and CONFIG.zhipuai_api_key != "YOUR_API_KEY": + elif CONFIG.zhipuai_api_key: llm = ZhiPuAIGPTAPI() else: raise RuntimeError("You should config a LLM configuration first") diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 064ec35ba..3161c0e88 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -13,6 +13,7 @@ from tenacity import ( ) from requests import ConnectionError +import openai import zhipuai from metagpt.config import CONFIG @@ -46,6 +47,7 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): def __init_zhipuai(self, config: CONFIG): assert config.zhipuai_api_key zhipuai.api_key = config.zhipuai_api_key + openai.api_key = zhipuai.api_key # due to use openai sdk, set the api_key but it will't be used. def _const_kwargs(self, messages: list[dict]) -> dict: kwargs = { From 7f1642720509cae1d2ce28d3f9a49c81e889a4ea Mon Sep 17 00:00:00 2001 From: seehi <6580@pm.me> Date: Wed, 22 Nov 2023 11:52:47 +0800 Subject: [PATCH 062/135] update readme --- README.md | 1 + docs/README_CN.md | 1 + docs/README_JA.md | 1 + 3 files changed, 3 insertions(+) diff --git a/README.md b/README.md index ead43c9e7..0ad622576 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ # If executing, ensure that NPM is installed on your system. Then install mermai detail installation please refer to [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) ### Docker installation +> Note: In the Windows, you need to replace "/opt/metagpt" with a directory that Docker has permission to create, such as "D:\Users\x\metagpt" ```bash # Step 1: Download metagpt official image and prepare config.yaml diff --git a/docs/README_CN.md b/docs/README_CN.md index 409bdc7af..ebdc63022 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -60,6 +60,7 @@ # 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid- 详细的安装请安装 [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) ### Docker安装 +> 注意：在Windows中，你需要将 "/opt/metagpt" 替换为Docker具有创建权限的目录，比如"D:\Users\x\metagpt" ```bash # 步骤1: 下载metagpt官方镜像并准备好config.yaml diff --git a/docs/README_JA.md b/docs/README_JA.md index 10cb7ee82..988dcde7a 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -163,6 +163,7 @@ # NPM がシステムにインストールされていることを確認して 注: この方法は pdf エクスポートに対応していません。 ### Docker によるインストール +> Windowsでは、"/opt/metagpt"をDockerが作成する権限を持つディレクトリに置き換える必要があります。例えば、"D:\Users\x\metagpt"などです。 ```bash # ステップ 1: metagpt 公式イメージをダウンロードし、config.yaml を準備する From 2bf8ef8c6ad18808447b827b6699e89650d7170c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 17:08:00 +0800 Subject: [PATCH 063/135] feat: RFC 135 --- metagpt/actions/design_api.py | 47 +++++++- metagpt/actions/prepare_documents.py | 38 +++++-- metagpt/actions/write_prd.py | 38 ++++++- metagpt/config.py | 3 +- metagpt/const.py | 5 + metagpt/environment.py | 9 +- metagpt/roles/product_manager.py | 7 +- metagpt/schema.py | 40 ++++++- metagpt/utils/dependency_file.py | 83 ++++++++++++++ metagpt/utils/file_repository.py | 116 ++++++++++++-------- metagpt/utils/git_repository.py | 15 ++- requirements.txt | 2 +- startup.py | 6 + tests/metagpt/utils/test_dependency_file.py | 64 +++++++++++ tests/metagpt/utils/test_file_repository.py | 10 +- tests/metagpt/utils/test_git_repository.py | 15 +++ 16 files changed, 416 insertions(+), 82 deletions(-) create mode 100644 metagpt/utils/dependency_file.py create mode 100644 tests/metagpt/utils/test_dependency_file.py diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 75df8b909..65d53364b 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,8 +11,9 @@ from typing import List from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT +from metagpt.const import PRDS_FILE_REPO, SYS_DESIGN_FILE_REPO, WORKSPACE_ROOT from metagpt.logs import logger +from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template from metagpt.utils.json_to_markdown import json_to_markdown @@ -202,7 +203,44 @@ class WriteDesign(Action): await self._save_prd(docs_path, resources_path, context) await self._save_system_design(docs_path, resources_path, system_design) - async def run(self, context, format=CONFIG.prompt_format): + async def run(self, with_messages, format=CONFIG.prompt_format): + # 通过git diff来识别docs/prds下哪些PRD文档发生了变动 + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) + changed_prds = prds_file_repo.changed_files + # 通过git diff来识别docs/system_designs下那些设计文档发生了变动； + system_design_file_repo = CONFIG.git_repo.new_file_repository(SYS_DESIGN_FILE_REPO) + changed_system_designs = system_design_file_repo.changed_files + + # 对于那些发生变动的PRD和设计文档，重新生成设计内容； + changed_files = Documents() + for filename in changed_prds.keys(): + prd = await prds_file_repo.get(filename) + old_system_design_doc = await system_design_file_repo.get(filename) + if not old_system_design_doc: + system_design = await self._run(context=prd.content) + doc = Document( + root_path=SYS_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() + ) + else: + doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) + await system_design_file_repo.save( + filename=filename, content=doc.content, dependencies={prd.root_relative_path} + ) + changed_files.docs[filename] = doc + + for filename in changed_system_designs.keys(): + if filename in changed_files.docs: + continue + prd_doc = await prds_file_repo.get(filename=filename) + old_system_design_doc = await system_design_file_repo.get(filename) + new_system_design_doc = await self._merge(prd_doc, old_system_design_doc) + await system_design_file_repo.save(filename=filename, content=new_system_design_doc.content) + changed_files.docs[filename] = new_system_design_doc + + # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 + return ActionOutput(content=changed_files.json(), instruct_content=changed_files) + + async def _run(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) # system_design = await self._aask(prompt) @@ -213,5 +251,8 @@ class WriteDesign(Action): "Python package name", system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), ) - await self._save(context, system_design) + # await self._save(context, system_design) return system_design + + async def _merge(self, prd_doc, system_design_doc): + return system_design_doc diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index b0185996b..c9b60ff27 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -7,19 +7,37 @@ @Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. RFC 135 2.2.3.5.1. """ -from metagpt.actions import Action + +from pathlib import Path + +from metagpt.actions import Action, ActionOutput +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME, WORKSPACE_ROOT +from metagpt.schema import Document +from metagpt.utils.file_repository import FileRepository +from metagpt.utils.git_repository import GitRepository class PrepareDocuments(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, with_message, **kwargs): - parent = self.context.get("parent") - if not parent: - raise ValueError("Invalid owner") - env = parent.get_env() - if env.git_repository: - return - env.git_repository = GitRepository() - env.git_repository.open(WORKS) + async def run(self, with_messages, **kwargs): + if CONFIG.git_repo: + docs_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) + doc = await docs_repo.get(REQUIREMENT_FILENAME) + return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) + + # Create and initialize the workspace folder, initialize the Git environment. + CONFIG.git_repo = GitRepository() + workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_file_name() + CONFIG.git_repo.open(local_path=workdir, auto_init=True) + + # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. + docs_file_repository = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) + doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) + await docs_file_repository.save(REQUIREMENT_FILENAME, content=doc.content) + + # Send a Message notification to the WritePRD action, instructing it to process requirements using + # `docs/requirement.txt` and `docs/prds/`. + return ActionOutput(content=doc.content, instruct_content=doc) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index bd04ca79e..a16d1ec06 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -10,7 +10,10 @@ from typing import List from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.logs import logger +from metagpt.schema import Document, Documents +from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template templates = { @@ -222,7 +225,34 @@ class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: + async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: + # 判断哪些需求文档需要重写：调LLM判断新增需求与prd是否相关，若相关就rewrite prd + docs_file_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) + requirement_doc = await docs_file_repo.get(REQUIREMENT_FILENAME) + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) + prd_docs = await prds_file_repo.get_all() + change_files = Documents() + for prd_doc in prd_docs: + if await self._is_relative_to(requirement_doc, prd_doc): + prd_doc = await self._merge(requirement_doc, prd_doc) + await prds_file_repo.save(filename=prd_doc.filename, content=prd_doc.content) + change_files.docs[prd_doc.filename] = prd_doc + # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd + if not change_files.docs: + prd = await self._run_new_requirement( + requirements=[requirement_doc.content], format=format, *args, **kwargs + ) + doc = Document( + root_path=PRDS_FILE_REPO, + filename=FileRepository.new_file_name() + ".json", + content=prd.instruct_content.json(), + ) + await prds_file_repo.save(filename=doc.filename, content=doc.content) + change_files.docs[doc.filename] = doc + # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 + return ActionOutput(content=change_files.json(), instruct_content=change_files) + + async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: sas = SearchAndSummarize() # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) rsp = "" @@ -239,3 +269,9 @@ class WritePRD(Action): # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) return prd + + async def _is_relative_to(self, doc1, doc2) -> bool: + return False + + async def _merge(self, doc1, doc2) -> Document: + pass diff --git a/metagpt/config.py b/metagpt/config.py index 27455d38d..51eed4fb8 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -46,7 +46,7 @@ class Config(metaclass=Singleton): self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -93,6 +93,7 @@ class Config(metaclass=Singleton): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + self.git_repo = None def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" diff --git a/metagpt/const.py b/metagpt/const.py index fa0ccc536..fc1c47b5b 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -49,3 +49,8 @@ MESSAGE_ROUTE_TO = "send_to" MESSAGE_ROUTE_CAUSE_BY = "cause_by" MESSAGE_META_ROLE = "role" MESSAGE_ROUTE_TO_ALL = "" + +REQUIREMENT_FILENAME = "requirement.txt" +DOCS_FILE_REPO = "docs" +PRDS_FILE_REPO = "docs/prds" +SYS_DESIGN_FILE_REPO = "docs/system_design" diff --git a/metagpt/environment.py b/metagpt/environment.py index df93a818b..b3c296dac 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -12,7 +12,7 @@ functionality is to be consolidated into the `Environment` class. """ import asyncio -from typing import Iterable, Optional, Set +from typing import Iterable, Set from pydantic import BaseModel, Field @@ -20,7 +20,6 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import is_subscribed -from metagpt.utils.git_repository import GitRepository class Environment(BaseModel): @@ -32,7 +31,6 @@ class Environment(BaseModel): roles: dict[str, Role] = Field(default_factory=dict) consumers: dict[Role, Set] = Field(default_factory=dict) history: str = Field(default="") # For debug - git_repository: Optional[GitRepository] = None class Config: arbitrary_types_allowed = True @@ -113,8 +111,3 @@ class Environment(BaseModel): def set_subscription(self, obj, tags): """Set the labels for message to be consumed by the object""" self.consumers[obj] = tags - - def dict(self, *args, **kwargs): - """Generate a dictionary representation of the model, optionally specifying which fields to include or - exclude.""" - return super(Environment, self).dict(exclude={"git_repository"}) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index c10aba6d1..81577ec2c 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -7,6 +7,7 @@ """ from metagpt.actions import BossRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.config import CONFIG from metagpt.roles import Role @@ -38,12 +39,12 @@ class ProductManager(Role): constraints (str): Constraints or limitations for the product manager. """ super().__init__(name, profile, goal, constraints) - self._init_actions([PrepareDocuments(context={"parent": self}), WritePRD]) - self._watch([BossRequirement]) + self._init_actions([PrepareDocuments, WritePRD]) + self._watch([BossRequirement, PrepareDocuments]) async def _think(self) -> None: """Decide what to do""" - if self._rc.env.git_repository: + if CONFIG.git_repo: self._set_state(1) else: self._set_state(0) diff --git a/metagpt/schema.py b/metagpt/schema.py index 82a0117ef..674091e4c 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -6,14 +6,16 @@ @File : schema.py @Modified By: mashenquan, 2023-10-31. According to Chapter 2.2.1 of RFC 116: Replanned the distribution of responsibilities and functional positioning of `Message` class attributes. +@Modified By: mashenquan, 2023/11/22. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. """ from __future__ import annotations import asyncio import json +import os.path from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError -from typing import List, Set, TypedDict +from typing import Dict, List, Optional, Set, TypedDict from pydantic import BaseModel, Field @@ -32,6 +34,42 @@ class RawMessage(TypedDict): role: str +class Document(BaseModel): + """ + Represents a document. + """ + + root_path: str + filename: str + content: Optional[str] = None + + def get_meta(self) -> Document: + """Get metadata of the document. + + :return: A new Document instance with the same root path and filename. + """ + + return Document(root_path=self.root_path, filename=self.filename) + + @property + def root_relative_path(self): + """Get relative path from root of git repository. + + :return: relative path from root of git repository. + """ + return os.path.join(self.root_path, self.filename) + + +class Documents(BaseModel): + """A class representing a collection of documents. + + Attributes: + docs (Dict[str, Document]): A dictionary mapping document names to Document instances. + """ + + docs: Dict[str, Document] = Field(default_factory=dict) + + class Message(BaseModel): """list[: ]""" diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py new file mode 100644 index 000000000..429027c7a --- /dev/null +++ b/metagpt/utils/dependency_file.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/22 +@Author : mashenquan +@File : dependency_file.py +@Desc: Implementation of the dependency file described in Section 2.2.3.2 of RFC 135. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Set + +import aiofiles + +from metagpt.logs import logger + + +class DependencyFile: + def __init__(self, workdir: Path | str): + self._dependencies = {} + self._filename = Path(workdir) / ".dependencies.json" + + async def load(self): + if not self._filename.exists(): + return + try: + async with aiofiles.open(str(self._filename), mode="r") as reader: + data = await reader.read() + self._dependencies = json.loads(data) + except Exception as e: + logger.error(f"Failed to load {str(self._filename)}, error:{e}") + + async def save(self): + try: + data = json.dumps(self._dependencies) + async with aiofiles.open(str(self._filename), mode="w") as writer: + await writer.write(data) + except Exception as e: + logger.error(f"Failed to save {str(self._filename)}, error:{e}") + + async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): + if persist: + await self.load() + + root = self._filename.parent + try: + key = Path(filename).relative_to(root) + except ValueError: + key = filename + + if dependencies: + relative_paths = [] + for i in dependencies: + try: + relative_paths.append(str(Path(i).relative_to(root))) + except ValueError: + relative_paths.append(str(i)) + self._dependencies[str(key)] = relative_paths + elif str(key) in self._dependencies: + del self._dependencies[str(key)] + + if persist: + await self.save() + + async def get(self, filename: Path | str, persist=False): + if persist: + await self.load() + + root = self._filename.parent + try: + key = Path(filename).relative_to(root) + except ValueError: + key = filename + return set(self._dependencies.get(str(key), {})) + + def delete_file(self): + self._filename.unlink(missing_ok=True) + + @property + def exists(self): + return self._filename.exists() diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index f4c36b5b7..7f07e4427 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -8,16 +8,29 @@ """ from __future__ import annotations -import json +import os +import uuid +from datetime import datetime from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Set import aiofiles from metagpt.logs import logger +from metagpt.schema import Document class FileRepository: + """A class representing a FileRepository associated with a Git repository. + + :param git_repo: The associated GitRepository instance. + :param relative_path: The relative path within the Git repository. + + Attributes: + _relative_path (Path): The relative path within the Git repository. + _git_repo (GitRepository): The associated GitRepository instance. + """ + def __init__(self, git_repo, relative_path: Path = Path(".")): """Initialize a FileRepository instance. @@ -26,16 +39,9 @@ class FileRepository: """ self._relative_path = relative_path self._git_repo = git_repo - self._dependencies: Dict[str, List[str]] = {} # Initializing self.workdir.mkdir(parents=True, exist_ok=True) - if self.dependency_path_name.exists(): - try: - with open(str(self.dependency_path_name), mode="r") as reader: - self._dependencies = json.load(reader) - except Exception as e: - logger.error(f"Failed to load {str(self.dependency_path_name)}, error:{e}") async def save(self, filename: Path | str, content, dependencies: List[str] = None): """Save content to a file and update its dependencies. @@ -44,59 +50,68 @@ class FileRepository: :param content: The content to be saved. :param dependencies: List of dependency filenames or paths. """ - path_name = self.workdir / filename - path_name.parent.mkdir(parents=True, exist_ok=True) - async with aiofiles.open(str(path_name), mode="w") as writer: + pathname = self.workdir / filename + pathname.parent.mkdir(parents=True, exist_ok=True) + async with aiofiles.open(str(pathname), mode="w") as writer: await writer.write(content) + logger.info(f"save to: {str(pathname)}") + if dependencies is not None: - await self.update_dependency(filename, dependencies) + dependency_file = await self._git_repo.get_dependency() + await dependency_file.update(pathname, set(dependencies)) + logger.info(f"update dependency: {str(pathname)}:{dependencies}") - async def get(self, filename: Path | str): - """Read the content of a file. - - :param filename: The filename or path within the repository. - :return: The content of the file. - """ - path_name = self.workdir / filename - async with aiofiles.open(str(path_name), mode="r") as reader: - return await reader.read() - - def get_dependency(self, filename: Path | str) -> List: + async def get_dependency(self, filename: Path | str) -> Set[str]: """Get the dependencies of a file. :param filename: The filename or path within the repository. - :return: List of dependency filenames or paths. + :return: Set of dependency filenames or paths. """ - key = str(filename) - return self._dependencies.get(key, []) + pathname = self.workdir / filename + dependency_file = await self._git_repo.get_dependency() + return await dependency_file.get(pathname) - def get_changed_dependency(self, filename: Path | str) -> List: + async def get_changed_dependency(self, filename: Path | str) -> Set[str]: """Get the dependencies of a file that have changed. :param filename: The filename or path within the repository. :return: List of changed dependency filenames or paths. """ - dependencies = self.get_dependency(filename=filename) + dependencies = await self.get_dependency(filename=filename) changed_files = self.changed_files - changed_dependent_files = [] + changed_dependent_files = set() for df in dependencies: if df in changed_files.keys(): - changed_dependent_files.append(df) + changed_dependent_files.add(df) return changed_dependent_files - async def update_dependency(self, filename, dependencies: List[str]): - """Update the dependencies of a file. + async def get(self, filename: Path | str) -> Document | None: + """Read the content of a file. :param filename: The filename or path within the repository. - :param dependencies: List of dependency filenames or paths. + :return: The content of the file. """ - self._dependencies[str(filename)] = dependencies + doc = Document(root_path=str(self.root_path), filename=str(filename)) + path_name = self.workdir / filename + if not path_name.exists(): + return None + async with aiofiles.open(str(path_name), mode="r") as reader: + doc.content = await reader.read() + return doc - async def save_dependency(self): - """Save the dependencies to a file.""" - data = json.dumps(self._dependencies) - with aiofiles.open(str(self.dependency_path_name), mode="w") as writer: - await writer.write(data) + async def get_all(self) -> List[Document]: + """Get the content of all files in the repository. + + :return: List of Document instances representing files. + """ + docs = [] + for root, dirs, files in os.walk(str(self.workdir)): + for file in files: + file_path = Path(root) / file + relative_path = file_path.relative_to(self.workdir) + doc = await self.get(relative_path) + docs.append(doc) + return docs @property def workdir(self): @@ -107,14 +122,9 @@ class FileRepository: return self._git_repo.workdir / self._relative_path @property - def dependency_path_name(self): - """Return the absolute path to the dependency file. - - :return: The absolute path to the dependency file. - """ - filename = ".dependencies.json" - path_name = self.workdir / filename - return path_name + def root_path(self): + """Return the relative path from git repository root""" + return self._relative_path @property def changed_files(self) -> Dict[str, str]: @@ -147,3 +157,13 @@ class FileRepository: continue children.append(str(f)) return children + + @staticmethod + def new_file_name(): + """Generate a new filename based on the current timestamp and a UUID suffix. + + :return: A new filename string. + """ + current_time = datetime.now().strftime("%Y%m%d%H%M%S") + guid_suffix = str(uuid.uuid4())[:8] + return f"{current_time}t{guid_suffix}" diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 6ae6a7900..a81b5c4ea 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -17,6 +17,7 @@ from git.repo import Repo from git.repo.fun import is_git_dir from metagpt.const import WORKSPACE_ROOT +from metagpt.utils.dependency_file import DependencyFile from metagpt.utils.file_repository import FileRepository @@ -47,6 +48,7 @@ class GitRepository: :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. """ self._repository = None + self._dependency = None if local_path: self.open(local_path=local_path, auto_init=auto_init) @@ -113,7 +115,7 @@ class GitRepository: :param local_path: The local path to check. :return: True if the directory is a Git repository, False otherwise. """ - git_dir = local_path / ".git" + git_dir = Path(local_path) / ".git" if git_dir.exists() and is_git_dir(git_dir): return True return False @@ -151,7 +153,7 @@ class GitRepository: self.add_change(self.changed_files) self.commit(comments) - def new_file_repository(self, relative_path: Path | str) -> FileRepository: + def new_file_repository(self, relative_path: Path | str = ".") -> FileRepository: """Create a new instance of FileRepository associated with this Git repository. :param relative_path: The relative path to the file repository within the Git repository. @@ -159,6 +161,15 @@ class GitRepository: """ return FileRepository(git_repo=self, relative_path=Path(relative_path)) + async def get_dependency(self) -> DependencyFile: + """Get the dependency file associated with the Git repository. + + :return: An instance of DependencyFile. + """ + if not self._dependency: + self._dependency = DependencyFile(workdir=self.workdir) + return self._dependency + if __name__ == "__main__": path = WORKSPACE_ROOT / "git" diff --git a/requirements.txt b/requirements.txt index c3b909e77..73a03d537 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,4 +44,4 @@ ta==0.10.2 semantic-kernel==0.3.13.dev0 wrapt==1.15.0 websocket-client==0.58.0 - +aiofiles==23.2.1 diff --git a/startup.py b/startup.py index e2a903c9b..d5a6bb07b 100644 --- a/startup.py +++ b/startup.py @@ -4,6 +4,7 @@ import asyncio import fire +from metagpt.config import CONFIG from metagpt.roles import ( Architect, Engineer, @@ -54,6 +55,7 @@ def main( code_review: bool = True, run_tests: bool = False, implement: bool = True, + project_path: str = None, ): """ We are a software startup comprised of AI. By investing in us, @@ -63,8 +65,12 @@ def main( a certain dollar amount to this AI company. :param n_round: :param code_review: Whether to use code review. + :param run_tests: Whether run unit tests. + :param implement: Whether to write codes. + :param project_path: The path of the old version project to improve. :return: """ + CONFIG.WORKDIR = project_path asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) diff --git a/tests/metagpt/utils/test_dependency_file.py b/tests/metagpt/utils/test_dependency_file.py new file mode 100644 index 000000000..ae4d40ea5 --- /dev/null +++ b/tests/metagpt/utils/test_dependency_file.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/22 +@Author : mashenquan +@File : test_dependency_file.py +@Desc: Unit tests for dependency_file.py +""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional, Set, Union + +import pytest +from pydantic import BaseModel + +from metagpt.utils.dependency_file import DependencyFile + + +@pytest.mark.asyncio +async def test_dependency_file(): + class Input(BaseModel): + x: Union[Path, str] + deps: Optional[Set[Union[Path, str]]] + key: Optional[Union[Path, str]] + want: Set[str] + + inputs = [ + Input(x="a/b.txt", deps={"c/e.txt", Path(__file__).parent / "d.txt"}, want={"c/e.txt", "d.txt"}), + Input( + x=Path(__file__).parent / "x/b.txt", + deps={"s/e.txt", Path(__file__).parent / "d.txt"}, + key="x/b.txt", + want={"s/e.txt", "d.txt"}, + ), + Input(x="f.txt", deps=None, want=set()), + Input(x="a/b.txt", deps=None, want=set()), + ] + + file = DependencyFile(workdir=Path(__file__).parent) + + for i in inputs: + await file.update(filename=i.x, dependencies=i.deps) + assert await file.get(filename=i.key or i.x) == i.want + + file2 = DependencyFile(workdir=Path(__file__).parent) + file2.delete_file() + assert not file.exists + await file2.update(filename="a/b.txt", dependencies={"c/e.txt", Path(__file__).parent / "d.txt"}, persist=False) + assert not file.exists + await file2.save() + assert file2.exists + + file1 = DependencyFile(workdir=Path(__file__).parent) + assert file1.exists + assert await file1.get("a/b.txt") == set() + await file1.load() + assert await file1.get("a/b.txt") == {"c/e.txt", "d.txt"} + file1.delete_file() + assert not file.exists + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_file_repository.py b/tests/metagpt/utils/test_file_repository.py index ac36f2320..a830b58aa 100644 --- a/tests/metagpt/utils/test_file_repository.py +++ b/tests/metagpt/utils/test_file_repository.py @@ -34,11 +34,13 @@ async def test_file_repo(): assert file_repo.workdir.exists() await file_repo.save("a.txt", "AAA") await file_repo.save("b.txt", "BBB", ["a.txt"]) - assert "AAA" == await file_repo.get("a.txt") - assert "BBB" == await file_repo.get("b.txt") - assert ["a.txt"] == file_repo.get_dependency("b.txt") + doc = await file_repo.get("a.txt") + assert "AAA" == doc.content + doc = await file_repo.get("b.txt") + assert "BBB" == doc.content + assert {"a.txt"} == await file_repo.get_dependency("b.txt") assert {"a.txt": ChangeType.UNTRACTED, "b.txt": ChangeType.UNTRACTED} == file_repo.changed_files - assert ["a.txt"] == file_repo.get_changed_dependency("b.txt") + assert {"a.txt"} == await file_repo.get_changed_dependency("b.txt") await file_repo.save("d/e.txt", "EEE") assert ["d/e.txt"] == file_repo.get_change_dir_files("d") diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index 0d1e3b791..23bebba7f 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -77,5 +77,20 @@ async def test_git1(): assert not local_path.exists() +@pytest.mark.asyncio +async def test_dependency_file(): + local_path = Path(__file__).parent / "git2" + repo, subdir = await mock_repo(local_path) + + dependancy_file = await repo.get_dependency() + assert not dependancy_file.exists + + await dependancy_file.update(filename="a/b.txt", dependencies={"c/d.txt", "e/f.txt"}) + assert dependancy_file.exists + + repo.delete_repository() + assert not dependancy_file.exists + + if __name__ == "__main__": pytest.main([__file__, "-s"]) From 9339eab20c95263549c8ad60a6bad087ab2cac46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 20:40:42 +0800 Subject: [PATCH 064/135] feat: archive --- metagpt/actions/design_api.py | 22 +++++++--- metagpt/actions/project_management.py | 63 ++++++++++++++++++++++++++- metagpt/const.py | 3 +- metagpt/utils/file_repository.py | 2 +- metagpt/utils/git_repository.py | 16 +++++++ 5 files changed, 97 insertions(+), 9 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 65d53364b..e7ee87fa2 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,7 +11,7 @@ from typing import List from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import PRDS_FILE_REPO, SYS_DESIGN_FILE_REPO, WORKSPACE_ROOT +from metagpt.const import PRDS_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser @@ -208,7 +208,7 @@ class WriteDesign(Action): prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) changed_prds = prds_file_repo.changed_files # 通过git diff来识别docs/system_designs下那些设计文档发生了变动； - system_design_file_repo = CONFIG.git_repo.new_file_repository(SYS_DESIGN_FILE_REPO) + system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_system_designs = system_design_file_repo.changed_files # 对于那些发生变动的PRD和设计文档，重新生成设计内容； @@ -219,7 +219,7 @@ class WriteDesign(Action): if not old_system_design_doc: system_design = await self._run(context=prd.content) doc = Document( - root_path=SYS_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() + root_path=SYSTEM_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() ) else: doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) @@ -234,7 +234,9 @@ class WriteDesign(Action): prd_doc = await prds_file_repo.get(filename=filename) old_system_design_doc = await system_design_file_repo.get(filename) new_system_design_doc = await self._merge(prd_doc, old_system_design_doc) - await system_design_file_repo.save(filename=filename, content=new_system_design_doc.content) + await system_design_file_repo.save( + filename=filename, content=new_system_design_doc.content, dependencies={prd_doc.root_relative_path} + ) changed_files.docs[filename] = new_system_design_doc # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 @@ -251,8 +253,18 @@ class WriteDesign(Action): "Python package name", system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), ) - # await self._save(context, system_design) + await self._rename_workspace(system_design) return system_design async def _merge(self, prd_doc, system_design_doc): return system_design_doc + + async def _rename_workspace(self, system_design): + if CONFIG.WORKDIR: # 已经指定了在旧版本上更新 + return + + if isinstance(system_design, ActionOutput): + ws_name = system_design.instruct_content.dict()["Python package name"] + else: + ws_name = CodeParser.parse_str(block="Python package name", text=system_design) + CONFIG.git_repo.rename_root(ws_name) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index b395fa64e..73481c780 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -5,11 +5,14 @@ @Author : alexanderwu @File : project_management.py """ +import json from typing import List +from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, WORKSPACE_ROOT +from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template from metagpt.utils.json_to_markdown import json_to_markdown @@ -178,13 +181,69 @@ class WriteTasks(Action): requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) - async def run(self, context, format=CONFIG.prompt_format): + async def run(self, with_messages, format=CONFIG.prompt_format): + system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) + changed_system_designs = system_design_file_repo.changed_files + + tasks_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) + changed_tasks = tasks_file_repo.changed_files + change_files = Documents() + # 根据docs/system_designs/下的git head diff识别哪些task文档需要重写 + for filename in changed_system_designs: + system_design_doc = await system_design_file_repo.get(filename) + task_doc = await tasks_file_repo.get(filename) + if task_doc: + task_doc = await self._merge(system_design_doc, task_doc) + else: + rsp = await self._run(system_design_doc.content) + task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) + await tasks_file_repo.save( + filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + ) + await self._update_requirements(task_doc) + change_files.docs[filename] = task_doc + + # 根据docs/tasks/下的git head diff识别哪些task文件被用户修改了，需要重写 + for filename in changed_tasks: + if filename in change_files.docs: + continue + system_design_doc = await system_design_file_repo.get(filename) + task_doc = await tasks_file_repo.get(filename) + task_doc = await self._merge(system_design_doc, task_doc) + await tasks_file_repo.save( + filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + ) + await self._update_requirements(task_doc) + change_files.docs[filename] = task_doc + + # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 + return ActionOutput(content=change_files.json(), instruct_content=change_files) + + async def _run(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) self._save(context, rsp) return rsp + async def _merge(self, system_design_doc, task_dock) -> Document: + return task_dock + + async def _update_requirements(self, doc): + m = json.loads(doc.content) + packages = set(m.get("Required Python third-party packages", set())) + file_repo = CONFIG.git_repo.new_file_repository() + filename = "requirements.txt" + requirement_doc = await file_repo.get(filename) + if not requirement_doc: + requirement_doc = Document(filename=filename, root_path=".", content="") + lines = requirement_doc.content.splitlines() + for pkg in lines: + if pkg == "": + continue + packages.add(pkg) + await file_repo.save(filename, content="\n".join(packages)) + class AssignTasks(Action): async def run(self, *args, **kwargs): diff --git a/metagpt/const.py b/metagpt/const.py index fc1c47b5b..63f39f4a8 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -53,4 +53,5 @@ MESSAGE_ROUTE_TO_ALL = "" REQUIREMENT_FILENAME = "requirement.txt" DOCS_FILE_REPO = "docs" PRDS_FILE_REPO = "docs/prds" -SYS_DESIGN_FILE_REPO = "docs/system_design" +SYSTEM_DESIGN_FILE_REPO = "docs/system_design" +TASK_FILE_REPO = "docs/tasks" diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 7f07e4427..ee6811209 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -166,4 +166,4 @@ class FileRepository: """ current_time = datetime.now().strftime("%Y%m%d%H%M%S") guid_suffix = str(uuid.uuid4())[:8] - return f"{current_time}t{guid_suffix}" + return f"{current_time}x{guid_suffix}" diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index a81b5c4ea..2a4fb4a4d 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -17,6 +17,7 @@ from git.repo import Repo from git.repo.fun import is_git_dir from metagpt.const import WORKSPACE_ROOT +from metagpt.logs import logger from metagpt.utils.dependency_file import DependencyFile from metagpt.utils.file_repository import FileRepository @@ -170,6 +171,21 @@ class GitRepository: self._dependency = DependencyFile(workdir=self.workdir) return self._dependency + def rename_root(self, new_dir_name): + """Rename the root directory of the Git repository. + + :param new_dir_name: The new name for the root directory. + """ + if self.workdir.name == new_dir_name: + return + new_path = self.workdir.parent / new_dir_name + if new_path.exists(): + logger.info(f"Delete directory {str(new_path)}") + shutil.rmtree(new_path) + self.workdir.rename(new_path) + logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") + self._repository = Repo(new_path) + if __name__ == "__main__": path = WORKSPACE_ROOT / "git" From e8131652de02a93454343d059dec02199f27b459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 21:45:44 +0800 Subject: [PATCH 065/135] refactor: write prd & system design --- metagpt/actions/design_api.py | 83 ++++++++++++++++++++++++++--------- metagpt/actions/write_prd.py | 70 ++++++++++++++++++++++------- metagpt/const.py | 5 +++ 3 files changed, 123 insertions(+), 35 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index e7ee87fa2..3bbde24ea 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -5,13 +5,21 @@ @Author : alexanderwu @File : design_api.py """ +import json import shutil from pathlib import Path from typing import List from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import PRDS_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, WORKSPACE_ROOT +from metagpt.const import ( + DATA_API_DESIGN_FILE_REPO, + PRDS_FILE_REPO, + SEQ_FLOW_FILE_REPO, + SYSTEM_DESIGN_FILE_REPO, + SYSTEM_DESIGN_PDF_FILE_REPO, + WORKSPACE_ROOT, +) from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser @@ -214,40 +222,29 @@ class WriteDesign(Action): # 对于那些发生变动的PRD和设计文档，重新生成设计内容； changed_files = Documents() for filename in changed_prds.keys(): - prd = await prds_file_repo.get(filename) - old_system_design_doc = await system_design_file_repo.get(filename) - if not old_system_design_doc: - system_design = await self._run(context=prd.content) - doc = Document( - root_path=SYSTEM_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() - ) - else: - doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) - await system_design_file_repo.save( - filename=filename, content=doc.content, dependencies={prd.root_relative_path} + doc = await self._update_system_design( + filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo ) changed_files.docs[filename] = doc for filename in changed_system_designs.keys(): if filename in changed_files.docs: continue - prd_doc = await prds_file_repo.get(filename=filename) - old_system_design_doc = await system_design_file_repo.get(filename) - new_system_design_doc = await self._merge(prd_doc, old_system_design_doc) - await system_design_file_repo.save( - filename=filename, content=new_system_design_doc.content, dependencies={prd_doc.root_relative_path} + doc = await self._update_system_design( + filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo ) - changed_files.docs[filename] = new_system_design_doc + changed_files.docs[filename] = doc # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=changed_files.json(), instruct_content=changed_files) - async def _run(self, context, format=CONFIG.prompt_format): + async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python package name" contain space, have to use setattr + # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python + # package name" contain space, have to use setattr setattr( system_design.instruct_content, "Python package name", @@ -268,3 +265,49 @@ class WriteDesign(Action): else: ws_name = CodeParser.parse_str(block="Python package name", text=system_design) CONFIG.git_repo.rename_root(ws_name) + + async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: + prd = await prds_file_repo.get(filename) + old_system_design_doc = await system_design_file_repo.get(filename) + if not old_system_design_doc: + system_design = await self._new_system_design(context=prd.content) + doc = Document( + root_path=SYSTEM_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() + ) + else: + doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) + await system_design_file_repo.save( + filename=filename, content=doc.content, dependencies={prd.root_relative_path} + ) + await self._save_data_api_design(doc) + await self._save_seq_flow(doc) + await self._save_pdf(doc) + return doc + + @staticmethod + async def _save_data_api_design(design_doc): + m = json.loads(design_doc.content) + data_api_design = m.get("Data structures and interface definitions") + if not data_api_design: + return + path = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO + if not path.exists(): + path.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(data_api_design, path / Path(design_doc).with_suffix(".mmd")) + + @staticmethod + async def _save_seq_flow(design_doc): + m = json.loads(design_doc.content) + seq_flow = m.get("Program call flow") + if not seq_flow: + return + path = CONFIG.git_repo.workdir / SEQ_FLOW_FILE_REPO + if not path.exists(): + path.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(seq_flow, path / Path(design_doc).with_suffix(".mmd")) + + @staticmethod + async def _save_pdf(design_doc): + m = json.loads(design_doc.content) + file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) + await file_repo.save(filename=design_doc.filename, content=json_to_markdown(m)) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index a16d1ec06..df35ec865 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -5,16 +5,28 @@ @Author : alexanderwu @File : write_prd.py """ +from __future__ import annotations + +import json +from pathlib import Path from typing import List from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG -from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME +from metagpt.const import ( + COMPETITIVE_ANALYSIS_FILE_REPO, + DOCS_FILE_REPO, + PRD_PDF_FILE_REPO, + PRDS_FILE_REPO, + REQUIREMENT_FILENAME, +) from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template +from metagpt.utils.json_to_markdown import json_to_markdown +from metagpt.utils.mermaid import mermaid_to_file templates = { "json": { @@ -233,22 +245,15 @@ class WritePRD(Action): prd_docs = await prds_file_repo.get_all() change_files = Documents() for prd_doc in prd_docs: - if await self._is_relative_to(requirement_doc, prd_doc): - prd_doc = await self._merge(requirement_doc, prd_doc) - await prds_file_repo.save(filename=prd_doc.filename, content=prd_doc.content) - change_files.docs[prd_doc.filename] = prd_doc + prd_doc = await self._update_prd(requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) + if not prd_doc: + continue + change_files.docs[prd_doc.filename] = prd_doc # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd if not change_files.docs: - prd = await self._run_new_requirement( - requirements=[requirement_doc.content], format=format, *args, **kwargs - ) - doc = Document( - root_path=PRDS_FILE_REPO, - filename=FileRepository.new_file_name() + ".json", - content=prd.instruct_content.json(), - ) - await prds_file_repo.save(filename=doc.filename, content=doc.content) - change_files.docs[doc.filename] = doc + prd_doc = await self._update_prd(requirement_doc, None, prds_file_repo) + if prd_doc: + change_files.docs[prd_doc.filename] = prd_doc # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 return ActionOutput(content=change_files.json(), instruct_content=change_files) @@ -275,3 +280,38 @@ class WritePRD(Action): async def _merge(self, doc1, doc2) -> Document: pass + + async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: + if not prd_doc: + prd = await self._run_new_requirement( + requirements=[requirement_doc.content], format=format, *args, **kwargs + ) + new_prd_doc = Document( + root_path=PRDS_FILE_REPO, + filename=FileRepository.new_file_name() + ".json", + content=prd.instruct_content.json(), + ) + elif await self._is_relative_to(requirement_doc, prd_doc): + new_prd_doc = await self._merge(requirement_doc, prd_doc) + else: + return None + await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content) + await self._save_competitive_analysis(new_prd_doc) + await self._save_pdf(new_prd_doc) + + @staticmethod + async def _save_competitive_analysis(prd_doc): + m = json.loads(prd_doc.content) + quadrant_chart = m.get("Competitive Quadrant Chart") + if not quadrant_chart: + return + path = CONFIG.git_repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO + if not path.exists(): + path.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(quadrant_chart, path / Path(prd_doc).with_suffix(".mmd")) + + @staticmethod + async def _save_pdf(prd_doc): + m = json.loads(prd_doc.content) + file_repo = CONFIG.git_repo.new_file_repository(PRD_PDF_FILE_REPO) + await file_repo.save(filename=prd_doc.filename, content=json_to_markdown(m)) diff --git a/metagpt/const.py b/metagpt/const.py index 63f39f4a8..b5ecad7cc 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -55,3 +55,8 @@ DOCS_FILE_REPO = "docs" PRDS_FILE_REPO = "docs/prds" SYSTEM_DESIGN_FILE_REPO = "docs/system_design" TASK_FILE_REPO = "docs/tasks" +COMPETITIVE_ANALYSIS_FILE_REPO = "resources/competitive_analysis" +DATA_API_DESIGN_FILE_REPO = "resources/data_api_design" +SEQ_FLOW_FILE_REPO = "resources/seq_flow" +SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" +PRD_PDF_FILE_REPO = "resources/prd" From 62d93517b48824c40209e447e8f76a59a7744d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 21:59:14 +0800 Subject: [PATCH 066/135] refactor: write prd & system design --- metagpt/actions/design_api.py | 21 +++++++++++++-------- metagpt/actions/write_prd.py | 8 ++++---- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 3bbde24ea..8fb926477 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -290,10 +290,9 @@ class WriteDesign(Action): data_api_design = m.get("Data structures and interface definitions") if not data_api_design: return - path = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO - if not path.exists(): - path.mkdir(parents=True, exists_ok=True) - await mermaid_to_file(data_api_design, path / Path(design_doc).with_suffix(".mmd")) + pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc).with_suffix(".mmd") + await WriteDesign._save_mermaid_file(data_api_design, pathname) + logger.info(f"Save class view to {str(pathname)}") @staticmethod async def _save_seq_flow(design_doc): @@ -301,13 +300,19 @@ class WriteDesign(Action): seq_flow = m.get("Program call flow") if not seq_flow: return - path = CONFIG.git_repo.workdir / SEQ_FLOW_FILE_REPO - if not path.exists(): - path.mkdir(parents=True, exists_ok=True) - await mermaid_to_file(seq_flow, path / Path(design_doc).with_suffix(".mmd")) + pathname = CONFIG.git_repo.workdir / SEQ_FLOW_FILE_REPO / Path(design_doc).with_suffix(".mmd") + await WriteDesign._save_mermaid_file(seq_flow, pathname) + logger.info(f"Saving sequence flow to {str(pathname)}") @staticmethod async def _save_pdf(design_doc): m = json.loads(design_doc.content) file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) await file_repo.save(filename=design_doc.filename, content=json_to_markdown(m)) + logger.info(f"Saving system design pdf to {design_doc.root_relative_path}") + + @staticmethod + async def _save_mermaid_file(data: str, pathname: Path): + if not pathname.parent.exists(): + pathname.parent.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(data, pathname) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index df35ec865..34001dec1 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -305,10 +305,10 @@ class WritePRD(Action): quadrant_chart = m.get("Competitive Quadrant Chart") if not quadrant_chart: return - path = CONFIG.git_repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO - if not path.exists(): - path.mkdir(parents=True, exists_ok=True) - await mermaid_to_file(quadrant_chart, path / Path(prd_doc).with_suffix(".mmd")) + pathname = CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc).with_suffix(".mmd") + if not pathname.parent.exists(): + pathname.parent.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(quadrant_chart, pathname) @staticmethod async def _save_pdf(prd_doc): From 369047e5586ef52cd21b9bc401630dfbba23fa29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 22:00:51 +0800 Subject: [PATCH 067/135] refactor: write prd & system design --- metagpt/actions/design_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 8fb926477..2c8c87558 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -290,7 +290,7 @@ class WriteDesign(Action): data_api_design = m.get("Data structures and interface definitions") if not data_api_design: return - pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc).with_suffix(".mmd") await WriteDesign._save_mermaid_file(data_api_design, pathname) logger.info(f"Save class view to {str(pathname)}") @@ -300,7 +300,7 @@ class WriteDesign(Action): seq_flow = m.get("Program call flow") if not seq_flow: return - pathname = CONFIG.git_repo.workdir / SEQ_FLOW_FILE_REPO / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc).with_suffix(".mmd") await WriteDesign._save_mermaid_file(seq_flow, pathname) logger.info(f"Saving sequence flow to {str(pathname)}") From 438fbe28c06aefee542bdd005941f801d4fe3e56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 11:29:09 +0800 Subject: [PATCH 068/135] refactor: save files --- metagpt/actions/design_api.py | 11 ++---- metagpt/actions/project_management.py | 54 +++++++++++++++++---------- metagpt/actions/write_prd.py | 23 +++++++----- metagpt/const.py | 1 + metagpt/utils/file_repository.py | 18 +++++++-- 5 files changed, 67 insertions(+), 40 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 2c8c87558..a8f89473d 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -290,7 +290,7 @@ class WriteDesign(Action): data_api_design = m.get("Data structures and interface definitions") if not data_api_design: return - pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc.filename).with_suffix("") await WriteDesign._save_mermaid_file(data_api_design, pathname) logger.info(f"Save class view to {str(pathname)}") @@ -300,19 +300,16 @@ class WriteDesign(Action): seq_flow = m.get("Program call flow") if not seq_flow: return - pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("") await WriteDesign._save_mermaid_file(seq_flow, pathname) logger.info(f"Saving sequence flow to {str(pathname)}") @staticmethod async def _save_pdf(design_doc): - m = json.loads(design_doc.content) file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) - await file_repo.save(filename=design_doc.filename, content=json_to_markdown(m)) - logger.info(f"Saving system design pdf to {design_doc.root_relative_path}") + await file_repo.save_pdf(doc=design_doc) @staticmethod async def _save_mermaid_file(data: str, pathname: Path): - if not pathname.parent.exists(): - pathname.parent.mkdir(parents=True, exists_ok=True) + pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(data, pathname) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 73481c780..686aa3689 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -11,7 +11,12 @@ from typing import List from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, WORKSPACE_ROOT +from metagpt.const import ( + SYSTEM_DESIGN_FILE_REPO, + TASK_FILE_REPO, + TASK_PDF_FILE_REPO, + WORKSPACE_ROOT, +) from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template @@ -190,46 +195,50 @@ class WriteTasks(Action): change_files = Documents() # 根据docs/system_designs/下的git head diff识别哪些task文档需要重写 for filename in changed_system_designs: - system_design_doc = await system_design_file_repo.get(filename) - task_doc = await tasks_file_repo.get(filename) - if task_doc: - task_doc = await self._merge(system_design_doc, task_doc) - else: - rsp = await self._run(system_design_doc.content) - task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) - await tasks_file_repo.save( - filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + task_doc = await self._update_tasks( + filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo ) - await self._update_requirements(task_doc) change_files.docs[filename] = task_doc # 根据docs/tasks/下的git head diff识别哪些task文件被用户修改了，需要重写 for filename in changed_tasks: if filename in change_files.docs: continue - system_design_doc = await system_design_file_repo.get(filename) - task_doc = await tasks_file_repo.get(filename) - task_doc = await self._merge(system_design_doc, task_doc) - await tasks_file_repo.save( - filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + task_doc = await self._update_tasks( + filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo ) - await self._update_requirements(task_doc) change_files.docs[filename] = task_doc # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=change_files.json(), instruct_content=change_files) - async def _run(self, context, format=CONFIG.prompt_format): + async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo): + system_design_doc = await system_design_file_repo.get(filename) + task_doc = await tasks_file_repo.get(filename) + if task_doc: + task_doc = await self._merge(system_design_doc=system_design_doc, task_dock=task_doc) + else: + rsp = await self._run_new_tasks(context=system_design_doc.content) + task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) + await tasks_file_repo.save( + filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + ) + await self._update_requirements(task_doc) + await self._save_pdf(task_doc=task_doc) + return task_doc + + async def _run_new_tasks(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) - self._save(context, rsp) + # self._save(context, rsp) return rsp async def _merge(self, system_design_doc, task_dock) -> Document: return task_dock - async def _update_requirements(self, doc): + @staticmethod + async def _update_requirements(doc): m = json.loads(doc.content) packages = set(m.get("Required Python third-party packages", set())) file_repo = CONFIG.git_repo.new_file_repository() @@ -244,6 +253,11 @@ class WriteTasks(Action): packages.add(pkg) await file_repo.save(filename, content="\n".join(packages)) + @staticmethod + async def _save_pdf(task_doc): + file_repo = CONFIG.git_repo.new_file_repository(TASK_PDF_FILE_REPO) + await file_repo.save_pdf(doc=task_doc) + class AssignTasks(Action): async def run(self, *args, **kwargs): diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 34001dec1..8b03ac29a 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -25,7 +25,6 @@ from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown from metagpt.utils.mermaid import mermaid_to_file templates = { @@ -245,13 +244,17 @@ class WritePRD(Action): prd_docs = await prds_file_repo.get_all() change_files = Documents() for prd_doc in prd_docs: - prd_doc = await self._update_prd(requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) + prd_doc = await self._update_prd( + requirement_doc=requirement_doc, prd_doc=prd_doc, prds_file_repo=prds_file_repo, *args, **kwargs + ) if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd if not change_files.docs: - prd_doc = await self._update_prd(requirement_doc, None, prds_file_repo) + prd_doc = await self._update_prd( + requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs + ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 @@ -283,9 +286,7 @@ class WritePRD(Action): async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: if not prd_doc: - prd = await self._run_new_requirement( - requirements=[requirement_doc.content], format=format, *args, **kwargs - ) + prd = await self._run_new_requirement(requirements=[requirement_doc.content], *args, **kwargs) new_prd_doc = Document( root_path=PRDS_FILE_REPO, filename=FileRepository.new_file_name() + ".json", @@ -298,6 +299,7 @@ class WritePRD(Action): await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content) await self._save_competitive_analysis(new_prd_doc) await self._save_pdf(new_prd_doc) + return new_prd_doc @staticmethod async def _save_competitive_analysis(prd_doc): @@ -305,13 +307,14 @@ class WritePRD(Action): quadrant_chart = m.get("Competitive Quadrant Chart") if not quadrant_chart: return - pathname = CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc).with_suffix(".mmd") + pathname = ( + CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + ) if not pathname.parent.exists(): - pathname.parent.mkdir(parents=True, exists_ok=True) + pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(quadrant_chart, pathname) @staticmethod async def _save_pdf(prd_doc): - m = json.loads(prd_doc.content) file_repo = CONFIG.git_repo.new_file_repository(PRD_PDF_FILE_REPO) - await file_repo.save(filename=prd_doc.filename, content=json_to_markdown(m)) + await file_repo.save_pdf(doc=prd_doc) diff --git a/metagpt/const.py b/metagpt/const.py index b5ecad7cc..7ee06ff7d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -60,3 +60,4 @@ DATA_API_DESIGN_FILE_REPO = "resources/data_api_design" SEQ_FLOW_FILE_REPO = "resources/seq_flow" SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" PRD_PDF_FILE_REPO = "resources/prd" +TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index ee6811209..62ba99d42 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -8,8 +8,8 @@ """ from __future__ import annotations +import json import os -import uuid from datetime import datetime from pathlib import Path from typing import Dict, List, Set @@ -18,6 +18,7 @@ import aiofiles from metagpt.logs import logger from metagpt.schema import Document +from metagpt.utils.json_to_markdown import json_to_markdown class FileRepository: @@ -165,5 +166,16 @@ class FileRepository: :return: A new filename string. """ current_time = datetime.now().strftime("%Y%m%d%H%M%S") - guid_suffix = str(uuid.uuid4())[:8] - return f"{current_time}x{guid_suffix}" + return current_time + # guid_suffix = str(uuid.uuid4())[:8] + # return f"{current_time}x{guid_suffix}" + + async def save_pdf(self, doc: Document): + """Save a Document as a PDF file. + + :param doc: The Document instance to be saved. + """ + m = json.loads(doc.content) + filename = Path(doc.filename).with_suffix(".md") + await self.save(filename=str(filename), content=json_to_markdown(m)) + logger.info(f"File Saved: {str(filename)}") From 2032a385426e44fea0154a292aa8e4b1e1a9be59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 17:49:38 +0800 Subject: [PATCH 069/135] feat: rewrite Engineer & WriteCode & WriteCodeReview --- metagpt/actions/write_code.py | 24 ++- metagpt/actions/write_code_review.py | 25 ++- metagpt/config.py | 1 + metagpt/roles/engineer.py | 266 +++++++++++---------------- metagpt/roles/qa_engineer.py | 7 - metagpt/schema.py | 7 + startup.py | 2 + 7 files changed, 152 insertions(+), 180 deletions(-) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index aeaa10aec..d4d33fe0c 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -7,13 +7,15 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` value of the `Message` object. """ +import json + from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions import WriteDesign from metagpt.actions.action import Action from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -from metagpt.schema import Message +from metagpt.schema import CodingContext from metagpt.utils.common import CodeParser, any_to_str PROMPT_TEMPLATE = """ @@ -46,7 +48,7 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc class WriteCode(Action): - def __init__(self, name="WriteCode", context: list[Message] = None, llm=None): + def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) def _is_invalid(self, filename): @@ -70,15 +72,19 @@ class WriteCode(Action): logger.info(f"Saving Code to {code_path}") @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def write_code(self, prompt): + async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) code = CodeParser.parse_code(block="", text=code_rsp) return code - async def run(self, context, filename): - prompt = PROMPT_TEMPLATE.format(context=context, filename=filename) - logger.info(f"Writing {filename}..") + async def run(self, *args, **kwargs) -> CodingContext: + m = json.loads(self.context.content) + coding_context = CodingContext(**m) + context = "\n".join( + [coding_context.design_doc.content, coding_context.task_doc.content, coding_context.code_doc.content] + ) + prompt = PROMPT_TEMPLATE.format(context=context, filename=self.context.filename) + logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) - # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) - # self._save(context, filename, code) - return code + coding_context.code_doc.content = code + return coding_context diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 76adca255..10e4aec3b 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -10,7 +10,7 @@ from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action from metagpt.logs import logger -from metagpt.schema import Message +from metagpt.schema import CodingContext from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -63,7 +63,7 @@ FORMAT_EXAMPLE = """ class WriteCodeReview(Action): - def __init__(self, name="WriteCodeReview", context: list[Message] = None, llm=None): + def __init__(self, name="WriteCodeReview", context=None, llm=None): super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) @@ -72,11 +72,18 @@ class WriteCodeReview(Action): code = CodeParser.parse_code(block="", text=code_rsp) return code - async def run(self, context, code, filename): - format_example = FORMAT_EXAMPLE.format(filename=filename) - prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example) - logger.info(f"Code review {filename}..") + async def run(self, *args, **kwargs) -> CodingContext: + format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) + context = "\n".join( + [self.context.design_doc.content, self.context.task_doc.content, self.context.code_doc.content] + ) + prompt = PROMPT_TEMPLATE.format( + context=context, + code=self.context.code_doc.content, + filename=self.context.code_doc.filename, + format_example=format_example, + ) + logger.info(f"Code review {self.context.code_doc.filename}..") code = await self.write_code(prompt) - # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) - # self._save(context, filename, code) - return code + self.context.code_doc.content = code + return self.context diff --git a/metagpt/config.py b/metagpt/config.py index 51eed4fb8..d059a6a29 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -94,6 +94,7 @@ class Config(metaclass=Singleton): self.prompt_format = self._get("PROMPT_FORMAT", "markdown") self.git_repo = None + self.src_workspace = None def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d23d23d55..8852d55f1 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -11,47 +11,20 @@ 3. Fix bug: Add logic for handling asynchronous message processing when messages are not ready. 4. Supplemented the external transmission of internal messages. """ -import asyncio -import shutil -from collections import OrderedDict +from __future__ import annotations + +import json from pathlib import Path -from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks +from metagpt.config import CONFIG +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import CodeParser, any_to_str +from metagpt.schema import CodingContext, Document, Documents, Message from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP -async def gather_ordered_k(coros, k) -> list: - tasks = OrderedDict() - results = [None] * len(coros) - done_queue = asyncio.Queue() - - for i, coro in enumerate(coros): - if len(tasks) >= k: - done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED) - for task in done: - index = tasks.pop(task) - await done_queue.put((index, task.result())) - task = asyncio.create_task(coro) - tasks[task] = i - - if tasks: - done, _ = await asyncio.wait(tasks.keys()) - for task in done: - index = tasks[task] - await done_queue.put((index, task.result())) - - while not done_queue.empty(): - index, result = await done_queue.get() - results[index] = result - - return results - - class Engineer(Role): """ Represents an Engineer role responsible for writing and possibly reviewing code. @@ -77,105 +50,19 @@ class Engineer(Role): ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) - self._init_actions([WriteCode]) self.use_code_review = use_code_review - if self.use_code_review: - self._init_actions([WriteCode, WriteCodeReview]) self._watch([WriteTasks]) self.todos = [] self.n_borg = n_borg - @classmethod - def parse_tasks(self, task_msg: Message) -> list[str]: - if task_msg.instruct_content: - return task_msg.instruct_content.dict().get("Task list") - return CodeParser.parse_file_list(block="Task list", text=task_msg.content) + @staticmethod + def _parse_tasks(task_msg: Document) -> list[str]: + m = json.loads(task_msg.content) + return m.get("Task list") - @classmethod - def parse_code(self, code_text: str) -> str: - return CodeParser.parse_code(block="", text=code_text) - - @classmethod - def parse_workspace(cls, system_design_msg: Message) -> str: - if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - - def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] - if not msg: - return WORKSPACE_ROOT / "src" - workspace = self.parse_workspace(msg) - # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace - - def recreate_workspace(self): - workspace = self.get_workspace() - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # The folder does not exist, but we don't care - workspace.mkdir(parents=True, exist_ok=True) - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() - filename = filename.replace('"', "").replace("\n", "") - file = workspace / filename - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) - return file - - async def _act_mp(self) -> Message: - # self.recreate_workspace() - todo_coros = [] - for todo in self.todos: - todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), - filename=todo, - ) - todo_coros.append(todo_coro) - - rsps = await gather_ordered_k(todo_coros, self.n_borg) - for todo, code_rsp in zip(self.todos, rsps): - _ = self.parse_code(code_rsp) - logger.info(todo) - logger.info(code_rsp) - # self.write_file(todo, code) - msg = Message(content=code_rsp, role=self.profile, cause_by=self._rc.todo) - self._rc.memory.add(msg) - self.publish_message(msg) - del self.todos[0] - - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message(content="all done.", role=self.profile, cause_by=self._rc.todo) - return msg - - async def _act_sp(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later - for todo in self.todos: - code = await WriteCode().run(context=self._rc.history, filename=todo) - # logger.info(todo) - # logger.info(code_rsp) - # code = self.parse_code(code_rsp) - file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=self._rc.todo) - self._rc.memory.add(msg) - self.publish_message(msg) - - code_msg = todo + FILENAME_CODE_SEP + str(file_path) - code_msg_all.append(code_msg) - - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message( - content=MSG_SEP.join(code_msg_all), - role=self.profile, - cause_by=self._rc.todo, - send_to="Edward", - ) - return msg - - async def _act_sp_precision(self) -> Message: + async def _act_sp_precision(self, review=False) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) for todo in self.todos: """ # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): @@ -184,30 +71,29 @@ class Engineer(Role): 3. Do we need other codes (currently needed)? TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. """ - context = [] - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - for m in msg: - context.append(m.content) - context_str = "\n".join(context) - # Write code - code = await WriteCode().run(context=context_str, filename=todo) + coding_context = await todo.run() # Code review - if self.use_code_review: + if review: try: - rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) - code = rewrite_code + coding_context = await WriteCodeReview(context=coding_context, llm=self._llm).run() except Exception as e: logger.error("code review failed!", e) pass - file_path = self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) + await src_file_repo.save( + coding_context.filename, + dependencies={coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path}, + content=coding_context.code_doc.content, + ) + msg = Message( + content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode + ) self._rc.memory.add(msg) self.publish_message(msg) - code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg = coding_context.filename + FILENAME_CODE_SEP + str(coding_context.code_doc.root_relative_path) code_msg_all.append(code_msg) - logger.info(f"Done {self.get_workspace()} generating.") + logger.info(f"Done {CONFIG.src_workspace} generating.") msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, @@ -218,22 +104,92 @@ class Engineer(Role): async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" - if not self._rc.todo: - return None - if self.use_code_review: - return await self._act_sp_precision() - return await self._act_sp() + return await self._act_sp_precision(review=self.use_code_review) - async def _observe(self) -> int: - ret = await super(Engineer, self)._observe() - if ret == 0: - return ret + async def _think(self) -> Action | None: + if not CONFIG.src_workspace: + CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name + # Prepare file repos + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_src_files = src_file_repo.changed_files + task_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) + changed_task_files = task_file_repo.changed_files + design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) - # Parse task lists - for message in self._rc.news: - if not message.cause_by == any_to_str(WriteTasks): + changed_files = Documents() + # 由上游变化导致的recode + for filename in changed_task_files: + design_doc = await design_file_repo.get(filename) + task_doc = await task_file_repo.get(filename) + task_list = self._parse_tasks(task_doc) + for task_filename in task_list: + old_code_doc = await src_file_repo.get(task_filename) + if not old_code_doc: + old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=task_filename, content="") + context = CodingContext( + filename=task_filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc + ) + coding_doc = Document( + root_path=str(src_file_repo.root_path), filename=task_filename, content=context.json() + ) + if task_filename in changed_files.docs: + logger.error( + f"Log to expose potential file name conflicts: {coding_doc.json()} & " + f"{changed_files.docs[task_filename].json()}" + ) + changed_files.docs[task_filename] = coding_doc + self.todos = [WriteCode(context=i, llm=self._llm) for i in changed_files.docs.values()] + # 用户直接修改的code + dependency = await CONFIG.git_repo.get_dependency() + for filename in changed_src_files: + if filename in changed_files.docs: continue - self.todos = self.parse_tasks(message) - return 1 + coding_doc = await self._new_coding_doc( + filename=filename, + src_file_repo=src_file_repo, + task_file_repo=task_file_repo, + design_file_repo=design_file_repo, + dependency=dependency, + ) + changed_files.docs[filename] = coding_doc + self.todos.append(WriteCode(context=coding_doc, llm=self._llm)) + # 仅单测 + if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files.docs: + context = await self._new_coding_context( + filename=CONFIG.REQA_FILENAME, + src_file_repo=src_file_repo, + task_file_repo=task_file_repo, + design_file_repo=design_file_repo, + dependency=dependency, + ) + self.publish_message(Message(content=context.json(), instruct_content=context, cause_by=WriteCode)) - return 0 + if self.todos: + self._rc.todo = self.todos[0] + return self._rc.todo # For agent store + + @staticmethod + async def _new_coding_context( + filename, src_file_repo, task_file_repo, design_file_repo, dependency + ) -> CodingContext: + old_code_doc = await src_file_repo.get(filename) + if not old_code_doc: + old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="") + dependencies = {Path(i) for i in dependency.get(old_code_doc.root_relative_path)} + task_doc = None + design_doc = None + for i in dependencies: + if str(i.parent) == TASK_FILE_REPO: + task_doc = task_file_repo.get(i.filename) + elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: + design_doc = design_file_repo.get(i.filename) + context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc) + return context + + @staticmethod + async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency): + context = await Engineer._new_coding_context( + filename, src_file_repo, task_file_repo, design_file_repo, dependency + ) + coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json()) + return coding_doc diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 760b65736..b57b64a7e 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -151,13 +151,6 @@ class QaEngineer(Role): ) self.publish_message(msg) - async def _observe(self) -> int: - await super()._observe() - self._rc.news = [ - msg for msg in self._rc.news if self.profile in msg.send_to - ] # only relevant msgs count as observed news - return len(self._rc.news) - async def _act(self) -> Message: if self.test_round > self.test_round_allowed: result_msg = Message( diff --git a/metagpt/schema.py b/metagpt/schema.py index 674091e4c..6a707af3e 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -238,3 +238,10 @@ class MessageQueue: logger.warning(f"JSON load failed: {v}, error:{e}") return q + + +class CodingContext(BaseModel): + filename: str + design_doc: Document + task_doc: Document + code_doc: Document diff --git a/startup.py b/startup.py index d5a6bb07b..1a59e7fa2 100644 --- a/startup.py +++ b/startup.py @@ -56,6 +56,7 @@ def main( run_tests: bool = False, implement: bool = True, project_path: str = None, + reqa_file: str = None, ): """ We are a software startup comprised of AI. By investing in us, @@ -71,6 +72,7 @@ def main( :return: """ CONFIG.WORKDIR = project_path + CONFIG.REQA_FILENAME = reqa_file asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) From ec3dd004af6000ec44ec6bb2cd6ed49d39e09ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 22:41:44 +0800 Subject: [PATCH 070/135] feat: Change the operation of transmitting file content during the QA process to transmitting file names instead. --- metagpt/actions/debug_error.py | 12 +-- metagpt/actions/run_code.py | 24 ++--- metagpt/actions/write_test.py | 16 +-- metagpt/const.py | 2 + metagpt/roles/engineer.py | 45 ++++---- metagpt/roles/qa_engineer.py | 178 +++++++++++++++----------------- metagpt/schema.py | 26 +++++ metagpt/utils/git_repository.py | 7 +- 8 files changed, 159 insertions(+), 151 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 304b1bc3e..a55f13dad 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -5,7 +5,6 @@ @Author : alexanderwu @File : debug_error.py """ -import re from metagpt.actions.action import Action from metagpt.logs import logger @@ -36,18 +35,17 @@ class DebugError(Action): # fixed_code = await self._aask(prompt) # return fixed_code - async def run(self, context): - if "PASS" in context: + async def run(self, *args, **kwargs) -> str: + if "PASS" in self.context.output: return "", "the original code works fine, no need to debug" - file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1) - + file_name = self.context.code_filename logger.info(f"Debug and rewrite {file_name}") - prompt = PROMPT_TEMPLATE.format(context=context) + prompt = PROMPT_TEMPLATE.format(context=self.context.output) rsp = await self._aask(prompt) code = CodeParser.parse_code(block="", text=rsp) - return file_name, code + return code diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index f69d2cd1a..f2d323f06 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -98,24 +98,22 @@ class RunCode(Action): stdout, stderr = process.communicate() return stdout.decode("utf-8"), stderr.decode("utf-8") - async def run( - self, code, mode="script", code_file_name="", test_code="", test_file_name="", command=[], **kwargs - ) -> str: - logger.info(f"Running {' '.join(command)}") - if mode == "script": - outs, errs = await self.run_script(command=command, **kwargs) - elif mode == "text": - outs, errs = await self.run_text(code=code) + async def run(self, *args, **kwargs) -> str: + logger.info(f"Running {' '.join(self.context.command)}") + if self.context.mode == "script": + outs, errs = await self.run_script(command=self.context.command, **kwargs) + elif self.context.mode == "text": + outs, errs = await self.run_text(code=self.context.code) logger.info(f"{outs=}") logger.info(f"{errs=}") context = CONTEXT.format( - code=code, - code_file_name=code_file_name, - test_code=test_code, - test_file_name=test_file_name, - command=" ".join(command), + code=self.context.code, + code_file_name=self.context.code_filename, + test_code=self.context.test_code, + test_file_name=self.context.test_filename, + command=" ".join(self.context.command), outs=outs[:500], # outs might be long but they are not important, truncate them to avoid token overflow errs=errs[:10000], # truncate errors to avoid token overflow ) diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 35ff36dc2..9a9671bab 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -6,7 +6,9 @@ @File : environment.py """ from metagpt.actions.action import Action +from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.schema import TestingContext from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -47,12 +49,12 @@ class WriteTest(Action): code = code_rsp return code - async def run(self, code_to_test, test_file_name, source_file_path, workspace): + async def run(self, *args, **kwargs) -> TestingContext: prompt = PROMPT_TEMPLATE.format( - code_to_test=code_to_test, - test_file_name=test_file_name, - source_file_path=source_file_path, - workspace=workspace, + code_to_test=self.context.code_doc.content, + test_file_name=self.context.test_doc.filename, + source_file_path=self.context.code_doc.root_relative_path, + workspace=CONFIG.git_repo.workdir, ) - code = await self.write_code(prompt) - return code + self.context.test_doc.content = await self.write_code(prompt) + return self.context diff --git a/metagpt/const.py b/metagpt/const.py index 7ee06ff7d..e97ffdb7d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -61,3 +61,5 @@ SEQ_FLOW_FILE_REPO = "resources/seq_flow" SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" PRD_PDF_FILE_REPO = "resources/prd" TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" +TEST_CODES_FILE_REPO = "tests" +OUTPUTS_FILE_REPO = "outputs" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 8852d55f1..89827a1ca 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -15,6 +15,7 @@ from __future__ import annotations import json from pathlib import Path +from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.config import CONFIG @@ -22,7 +23,6 @@ from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import CodingContext, Document, Documents, Message -from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP class Engineer(Role): @@ -60,8 +60,8 @@ class Engineer(Role): m = json.loads(task_msg.content) return m.get("Task list") - async def _act_sp_precision(self, review=False) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + async def _act_sp_precision(self, review=False) -> Set[str]: + changed_files = set() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) for todo in self.todos: """ @@ -88,23 +88,26 @@ class Engineer(Role): content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode ) self._rc.memory.add(msg) - self.publish_message(msg) - code_msg = coding_context.filename + FILENAME_CODE_SEP + str(coding_context.code_doc.root_relative_path) - code_msg_all.append(code_msg) - - logger.info(f"Done {CONFIG.src_workspace} generating.") - msg = Message( - content=MSG_SEP.join(code_msg_all), - role=self.profile, - cause_by=self._rc.todo, - send_to="Edward", - ) - return msg + changed_files.add(coding_context.code_doc.filename) + return changed_files async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" - return await self._act_sp_precision(review=self.use_code_review) + changed_files = await self._act_sp_precision(review=self.use_code_review) + # 仅单测 + if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: + changed_files.add(CONFIG.REQA_FILENAME) + + from metagpt.roles import QaEngineer # 避免循环引用 + + msg = Message( + content="\n".join(changed_files), + role=self.profile, + cause_by=WriteCodeReview if self.use_code_review else WriteCode, + send_to=QaEngineer, + ) + return msg async def _think(self) -> Action | None: if not CONFIG.src_workspace: @@ -153,16 +156,6 @@ class Engineer(Role): ) changed_files.docs[filename] = coding_doc self.todos.append(WriteCode(context=coding_doc, llm=self._llm)) - # 仅单测 - if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files.docs: - context = await self._new_coding_context( - filename=CONFIG.REQA_FILENAME, - src_file_repo=src_file_repo, - task_file_repo=task_file_repo, - design_file_repo=design_file_repo, - dependency=dependency, - ) - self.publish_message(Message(content=context.json(), instruct_content=context, cause_by=WriteCode)) if self.todos: self._rc.todo = self.todos[0] diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index b57b64a7e..1520a830a 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -7,23 +7,15 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data type of the `cause_by` value in the `Message` to a string, and utilize the new message filtering feature. """ -import os -from pathlib import Path +import json -from metagpt.actions import ( - DebugError, - RunCode, - WriteCode, - WriteCodeReview, - WriteDesign, - WriteTest, -) -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest +from metagpt.config import CONFIG +from metagpt.const import OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import CodeParser, any_to_str_set, parse_recipient -from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP +from metagpt.schema import Document, Message, RunCodeContext, TestingContext +from metagpt.utils.common import CodeParser, any_to_str_set class QaEngineer(Role): @@ -49,107 +41,98 @@ class QaEngineer(Role): return system_design_msg.instruct_content.dict().get("Python package name") return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - def get_workspace(self, return_proj_dir=True) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] - if not msg: - return WORKSPACE_ROOT / "src" - workspace = self.parse_workspace(msg) - # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. - if return_proj_dir: - return WORKSPACE_ROOT / workspace - # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() / "tests" - file = workspace / filename - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) - async def _write_test(self, message: Message) -> None: - code_msgs = message.content.split(MSG_SEP) - # result_msg_all = [] - for code_msg in code_msgs: + changed_files = message.content.splitlines() + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) + for filename in changed_files: # write tests - file_name, file_path = code_msg.split(FILENAME_CODE_SEP) - code_to_test = open(file_path, "r").read() - if "test" in file_name: - continue # Engineer might write some test files, skip testing a test file - test_file_name = "test_" + file_name - test_file_path = self.get_workspace() / "tests" / test_file_name - logger.info(f"Writing {test_file_name}..") - test_code = await WriteTest().run( - code_to_test=code_to_test, - test_file_name=test_file_name, - # source_file_name=file_name, - source_file_path=file_path, - workspace=self.get_workspace(), + if not filename or "test" in filename: + continue + code_doc = await src_file_repo.get(filename) + test_doc = await tests_file_repo.get("test_" + code_doc.filename) + if not test_doc: + test_doc = Document( + root_path=str(tests_file_repo.root_path), filename="test_" + code_doc.filename, content="" + ) + logger.info(f"Writing {test_doc.filename}..") + context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc) + context = await WriteTest(context=context, llm=self._llm).run() + await tests_file_repo.save( + filename=context.test_doc.filename, + content=context.test_doc.content, + dependencies={context.code_doc.root_relative_path}, ) - self.write_file(test_file_name, test_code) # prepare context for run tests in next round - command = ["python", f"tests/{test_file_name}"] - file_info = { - "file_name": file_name, - "file_path": str(file_path), - "test_file_name": test_file_name, - "test_file_path": str(test_file_path), - "command": command, - } + run_code_context = RunCodeContext( + command=["python", context.test_doc.root_relative_path], + code_filename=context.code_doc.filename, + test_filename=context.test_doc.filename, + working_directory=str(CONFIG.git_repo.workdir), + additional_python_paths=[CONFIG.src_workspace], + ) + msg = Message( - content=str(file_info), + content=run_code_context.json(), role=self.profile, cause_by=WriteTest, - sent_from=self.profile, - send_to=self.profile, + sent_from=self, + send_to=self, ) self.publish_message(msg) - logger.info(f"Done {self.get_workspace()}/tests generating.") + logger.info(f"Done {str(tests_file_repo.workdir)} generating.") async def _run_code(self, msg): - file_info = eval(msg.content) - development_file_path = file_info["file_path"] - test_file_path = file_info["test_file_path"] - if not os.path.exists(development_file_path) or not os.path.exists(test_file_path): + m = json.loads(msg.content) + run_code_context = RunCodeContext(**m) + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + src_doc = await src_file_repo.get(run_code_context.code_filename) + if not src_doc: return - - development_code = open(development_file_path, "r").read() - test_code = open(test_file_path, "r").read() - proj_dir = self.get_workspace() - development_code_dir = self.get_workspace(return_proj_dir=False) - - result_msg = await RunCode().run( - mode="script", - code=development_code, - code_file_name=file_info["file_name"], - test_code=test_code, - test_file_name=file_info["test_file_name"], - command=file_info["command"], - working_directory=proj_dir, # workspace/package_name, will run tests/test_xxx.py here - additional_python_paths=[development_code_dir], # workspace/package_name/package_name, - # import statement inside package code needs this + test_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) + test_doc = await test_file_repo.get(run_code_context.test_filename) + if not test_doc: + return + run_code_context.code = src_doc.content + run_code_context.test_code = test_doc.content + result_msg = await RunCode(context=run_code_context, llm=self._llm).run() + outputs_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) + run_code_context.output_filename = run_code_context.test_filename + ".log" + await outputs_file_repo.save( + filename=run_code_context.output_filename, + content=result_msg, + dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, + ) + run_code_context.code = None + run_code_context.test_code = None + msg = Message( + content=run_code_context.json(), role=self.profile, cause_by=RunCode, sent_from=self, send_to=self ) - - recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself - content = str(file_info) + FILENAME_CODE_SEP + result_msg - msg = Message(content=content, role=self.profile, cause_by=RunCode, sent_from=self.profile, send_to=recipient) self.publish_message(msg) async def _debug_error(self, msg): - file_info, context = msg.content.split(FILENAME_CODE_SEP) - file_name, code = await DebugError().run(context) - if file_name: - self.write_file(file_name, code) - recipient = msg.sent_from # send back to the one who ran the code for another run, might be one's self - msg = Message( - content=file_info, - role=self.profile, - cause_by=DebugError, - sent_from=self.profile, - send_to=recipient, - ) - self.publish_message(msg) + m = json.loads(msg.context) + run_code_context = RunCodeContext(**m) + output_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) + output_doc = await output_file_repo.get(run_code_context.output_filename) + if not output_doc: + return + run_code_context.output = output_doc.content + code = await DebugError(context=run_code_context, llm=self._llm).run() + src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + await src_file_repo.save(filename=run_code_context.code_filename, content=code) + run_code_context.output = None + run_code_context.output_filename = None + msg = Message( + content=run_code_context.json(), + role=self.profile, + cause_by=DebugError, + sent_from=self, + send_to=self, + ) + self.publish_message(msg) async def _act(self) -> Message: if self.test_round > self.test_round_allowed: @@ -182,5 +165,6 @@ class QaEngineer(Role): role=self.profile, cause_by=WriteTest, sent_from=self.profile, + send_to="", ) return result_msg diff --git a/metagpt/schema.py b/metagpt/schema.py index 6a707af3e..5cc7cdb2d 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -19,6 +19,7 @@ from typing import Dict, List, Optional, Set, TypedDict from pydantic import BaseModel, Field +from metagpt.config import CONFIG from metagpt.const import ( MESSAGE_ROUTE_CAUSE_BY, MESSAGE_ROUTE_FROM, @@ -59,6 +60,12 @@ class Document(BaseModel): """ return os.path.join(self.root_path, self.filename) + @property + def full_path(self): + if not CONFIG.git_repo: + return None + return str(CONFIG.git_repo.workdir / self.root_path / self.filename) + class Documents(BaseModel): """A class representing a collection of documents. @@ -245,3 +252,22 @@ class CodingContext(BaseModel): design_doc: Document task_doc: Document code_doc: Document + + +class TestingContext(BaseModel): + filename: str + code_doc: Document + test_doc: Document + + +class RunCodeContext(BaseModel): + mode: str = "script" + code: Optional[str] + code_filename: str = "" + test_code: Optional[str] + test_filename: str = "" + command: List[str] = Field(default_factory=list) + working_directory: str = "" + additional_python_paths: List[str] = Field(default_factory=list) + output_filename: Optional[str] + output: Optional[str] diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 2a4fb4a4d..c2eb2360e 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -160,7 +160,12 @@ class GitRepository: :param relative_path: The relative path to the file repository within the Git repository. :return: A new instance of FileRepository. """ - return FileRepository(git_repo=self, relative_path=Path(relative_path)) + path = Path(relative_path) + try: + path = path.relative_to(self.workdir) + except ValueError: + path = relative_path + return FileRepository(git_repo=self, relative_path=Path(path)) async def get_dependency(self) -> DependencyFile: """Get the dependency file associated with the Git repository. From 6d77cd89c86c4dd01007933e2f19352ef32d7dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 22:54:22 +0800 Subject: [PATCH 071/135] refactor: delete useless codes --- metagpt/roles/qa_engineer.py | 8 +------- metagpt/roles/role.py | 4 ---- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 7a2b7cbd4..6f0738294 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -15,7 +15,7 @@ from metagpt.const import OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext -from metagpt.utils.common import CodeParser, any_to_str_set +from metagpt.utils.common import any_to_str_set class QaEngineer(Role): @@ -35,12 +35,6 @@ class QaEngineer(Role): self.test_round = 0 self.test_round_allowed = test_round_allowed - @classmethod - def parse_workspace(cls, system_design_msg: Message) -> str: - if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name") - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - async def _write_test(self, message: Message) -> None: changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index d1e65a4e0..2e3bcbbd5 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -164,10 +164,6 @@ class Role: if env: env.set_subscription(self, self._subscription) - def get_env(self): - """Return the environment in which the role works.""" - return self._rc.env - @property def profile(self): """Get the role description (position)""" From 10d9f33150a5e4b96bf904098db7ebaa7dc2aeca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 23:04:41 +0800 Subject: [PATCH 072/135] refactor: use MESSAGE_ROUTE_TO_NONE --- metagpt/const.py | 1 + metagpt/roles/qa_engineer.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index e97ffdb7d..311712013 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -49,6 +49,7 @@ MESSAGE_ROUTE_TO = "send_to" MESSAGE_ROUTE_CAUSE_BY = "cause_by" MESSAGE_META_ROLE = "role" MESSAGE_ROUTE_TO_ALL = "" +MESSAGE_ROUTE_TO_NONE = "" REQUIREMENT_FILENAME = "requirement.txt" DOCS_FILE_REPO = "docs" diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 6f0738294..eac30413a 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -11,7 +11,7 @@ import json from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest from metagpt.config import CONFIG -from metagpt.const import OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO +from metagpt.const import MESSAGE_ROUTE_TO_NONE, OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext @@ -66,7 +66,6 @@ class QaEngineer(Role): working_directory=str(CONFIG.git_repo.workdir), additional_python_paths=[CONFIG.src_workspace], ) - msg = Message( content=run_code_context.json(), role=self.profile, @@ -135,7 +134,7 @@ class QaEngineer(Role): role=self.profile, cause_by=WriteTest, sent_from=self.profile, - send_to="", + send_to=MESSAGE_ROUTE_TO_NONE, ) return result_msg @@ -160,6 +159,6 @@ class QaEngineer(Role): role=self.profile, cause_by=WriteTest, sent_from=self.profile, - send_to="", + send_to=MESSAGE_ROUTE_TO_NONE, ) return result_msg From 7ef9fddc6413f1ef94fcf1a683c8ae071f55e436 Mon Sep 17 00:00:00 2001 From: better629 Date: Fri, 24 Nov 2023 10:06:42 +0800 Subject: [PATCH 073/135] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ead43c9e7..fc8781014 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ ### Contact Information ## Citation -For now, cite the [Arxiv paper](https://arxiv.org/abs/2308.00352): +For now, cite the [arXiv paper](https://arxiv.org/abs/2308.00352): ```bibtex @misc{hong2023metagpt, From 7a4187001560f9f4db73052f0912abb3a309b527 Mon Sep 17 00:00:00 2001 From: better629 Date: Fri, 24 Nov 2023 10:07:12 +0800 Subject: [PATCH 074/135] Update README_CN.md --- docs/README_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README_CN.md b/docs/README_CN.md index 409bdc7af..6721ca9ca 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -120,7 +120,7 @@ ### 联系信息 ## 引用 -引用 [Arxiv paper](https://arxiv.org/abs/2308.00352): +引用 [arXiv paper](https://arxiv.org/abs/2308.00352): ```bibtex @misc{hong2023metagpt, From 9e97e6d3d0da67e220c263f23ec54cdc398bdcc2 Mon Sep 17 00:00:00 2001 From: better629 Date: Fri, 24 Nov 2023 10:07:38 +0800 Subject: [PATCH 075/135] Update README_JA.md --- docs/README_JA.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/README_JA.md b/docs/README_JA.md index 10cb7ee82..a38b92f5b 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -299,7 +299,7 @@ ## クイックスタート ## 引用 -現時点では、[Arxiv 論文](https://arxiv.org/abs/2308.00352)を引用してください: +現時点では、[arXiv 論文](https://arxiv.org/abs/2308.00352)を引用してください: ```bibtex @misc{hong2023metagpt, From 75dcc8d5341f6eda98ced66a032369953c75445c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 13:30:00 +0800 Subject: [PATCH 076/135] fixbug: DebugError --- metagpt/actions/debug_error.py | 5 ++- metagpt/actions/run_code.py | 25 +++++++++-- metagpt/roles/qa_engineer.py | 77 +++++++++++++++++--------------- metagpt/schema.py | 24 ++++++++++ metagpt/utils/common.py | 8 +++- metagpt/utils/file_repository.py | 11 ++++- 6 files changed, 107 insertions(+), 43 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index a55f13dad..7a12e18f8 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : debug_error.py """ +import re from metagpt.actions.action import Action from metagpt.logs import logger @@ -36,7 +37,9 @@ class DebugError(Action): # return fixed_code async def run(self, *args, **kwargs) -> str: - if "PASS" in self.context.output: + pattern = r"Ran (\d+) tests in ([\d.]+)s\n\nOK" + matches = re.search(pattern, self.context.output) + if matches: return "", "the original code works fine, no need to debug" file_name = self.context.code_filename diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index f2d323f06..b244577a7 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -51,8 +51,14 @@ CONTEXT = """ ## Running Command {command} ## Running Output -standard output: {outs}; -standard errors: {errs}; +standard output: +```text +{outs} +``` +standard errors: +```text +{errs} +``` """ @@ -84,10 +90,19 @@ class RunCode(Action): additional_python_paths = ":".join(additional_python_paths) env["PYTHONPATH"] = additional_python_paths + ":" + env.get("PYTHONPATH", "") + install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] + logger.info(" ".join(install_command)) + subprocess.run(install_command, check=True, cwd=working_directory, env=env) + + install_pytest_command = ["python", "-m", "pip", "install", "pytest"] + logger.info(" ".join(install_pytest_command)) + subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) + # Start the subprocess process = subprocess.Popen( command, cwd=working_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env ) + logger.info(" ".join(command)) try: # Wait for the process to complete, with a timeout @@ -101,7 +116,11 @@ class RunCode(Action): async def run(self, *args, **kwargs) -> str: logger.info(f"Running {' '.join(self.context.command)}") if self.context.mode == "script": - outs, errs = await self.run_script(command=self.context.command, **kwargs) + outs, errs = await self.run_script( + command=self.context.command, + working_directory=self.context.working_directory, + additional_python_paths=self.context.additional_python_paths, + ) elif self.context.mode == "text": outs, errs = await self.run_text(code=self.context.code) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index eac30413a..f950efef4 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -7,15 +7,13 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data type of the `cause_by` value in the `Message` to a string, and utilize the new message filtering feature. """ -import json - from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest from metagpt.config import CONFIG from metagpt.const import MESSAGE_ROUTE_TO_NONE, OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext -from metagpt.utils.common import any_to_str_set +from metagpt.utils.common import any_to_str_set, parse_recipient class QaEngineer(Role): @@ -64,68 +62,76 @@ class QaEngineer(Role): code_filename=context.code_doc.filename, test_filename=context.test_doc.filename, working_directory=str(CONFIG.git_repo.workdir), - additional_python_paths=[CONFIG.src_workspace], + additional_python_paths=[str(CONFIG.src_workspace)], ) - msg = Message( - content=run_code_context.json(), - role=self.profile, - cause_by=WriteTest, - sent_from=self, - send_to=self, + self.publish_message( + Message( + content=run_code_context.json(), + role=self.profile, + cause_by=WriteTest, + sent_from=self, + send_to=self, + ) ) - self.publish_message(msg) logger.info(f"Done {str(tests_file_repo.workdir)} generating.") async def _run_code(self, msg): - m = json.loads(msg.content) - run_code_context = RunCodeContext(**m) - src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - src_doc = await src_file_repo.get(run_code_context.code_filename) + run_code_context = RunCodeContext.loads(msg.content) + src_doc = await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).get(run_code_context.code_filename) if not src_doc: return - test_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) - test_doc = await test_file_repo.get(run_code_context.test_filename) + test_doc = await CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get(run_code_context.test_filename) if not test_doc: return run_code_context.code = src_doc.content run_code_context.test_code = test_doc.content result_msg = await RunCode(context=run_code_context, llm=self._llm).run() - outputs_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) - run_code_context.output_filename = run_code_context.test_filename + ".log" - await outputs_file_repo.save( + run_code_context.output_filename = run_code_context.test_filename + ".md" + await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).save( filename=run_code_context.output_filename, content=result_msg, dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, ) run_code_context.code = None run_code_context.test_code = None - msg = Message( - content=run_code_context.json(), role=self.profile, cause_by=RunCode, sent_from=self, send_to=self + recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself + mappings = { + "Engineer": "Alex", + "QaEngineer": "Edward", + } + self.publish_message( + Message( + content=run_code_context.json(), + role=self.profile, + cause_by=RunCode, + sent_from=self, + send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE), + ) ) - self.publish_message(msg) async def _debug_error(self, msg): - m = json.loads(msg.context) - run_code_context = RunCodeContext(**m) + run_code_context = RunCodeContext.loads(msg.content) output_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) output_doc = await output_file_repo.get(run_code_context.output_filename) if not output_doc: return run_code_context.output = output_doc.content code = await DebugError(context=run_code_context, llm=self._llm).run() - src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - await src_file_repo.save(filename=run_code_context.code_filename, content=code) + await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).save( + filename=run_code_context.code_filename, content=code + ) run_code_context.output = None run_code_context.output_filename = None - msg = Message( - content=run_code_context.json(), - role=self.profile, - cause_by=DebugError, - sent_from=self, - send_to=self, + self.publish_message( + Message( + content=run_code_context.json(), + role=self.profile, + cause_by=DebugError, + sent_from=self, + send_to=self, + ) ) - self.publish_message(msg) async def _act(self) -> Message: if self.test_round > self.test_round_allowed: @@ -154,11 +160,10 @@ class QaEngineer(Role): # I ran my test code, time to fix bugs, if any await self._debug_error(msg) self.test_round += 1 - result_msg = Message( + return Message( content=f"Round {self.test_round} of tests done", role=self.profile, cause_by=WriteTest, sent_from=self.profile, send_to=MESSAGE_ROUTE_TO_NONE, ) - return result_msg diff --git a/metagpt/schema.py b/metagpt/schema.py index 5cc7cdb2d..53a22f0e6 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -253,12 +253,28 @@ class CodingContext(BaseModel): task_doc: Document code_doc: Document + @staticmethod + def loads(val: str) -> CodingContext | None: + try: + m = json.loads(val) + return CodingContext(**m) + except Exception: + return None + class TestingContext(BaseModel): filename: str code_doc: Document test_doc: Document + @staticmethod + def loads(val: str) -> TestingContext | None: + try: + m = json.loads(val) + return TestingContext(**m) + except Exception: + return None + class RunCodeContext(BaseModel): mode: str = "script" @@ -271,3 +287,11 @@ class RunCodeContext(BaseModel): additional_python_paths: List[str] = Field(default_factory=list) output_filename: Optional[str] output: Optional[str] + + @staticmethod + def loads(val: str) -> RunCodeContext | None: + try: + m = json.loads(val) + return RunCodeContext(**m) + except Exception: + return None diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 798acf214..9002a8dfb 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -304,7 +304,13 @@ def print_members(module, indent=0): def parse_recipient(text): pattern = r"## Send To:\s*([A-Za-z]+)\s*?" # hard code for now recipient = re.search(pattern, text) - return recipient.group(1) if recipient else "" + if recipient: + return recipient.group(1) + pattern = r"Send To:\s*([A-Za-z]+)\s*?" + recipient = re.search(pattern, text) + if recipient: + return recipient.group(1) + return "" def get_class_name(cls) -> str: diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 62ba99d42..8de4bdf5b 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -96,8 +96,15 @@ class FileRepository: path_name = self.workdir / filename if not path_name.exists(): return None - async with aiofiles.open(str(path_name), mode="r") as reader: - doc.content = await reader.read() + try: + async with aiofiles.open(str(path_name), mode="r") as reader: + doc.content = await reader.read() + except FileNotFoundError as e: + logger.info(f"open {str(path_name)} failed:{e}") + return None + except Exception as e: + logger.info(f"open {str(path_name)} failed:{e}") + return None return doc async def get_all(self) -> List[Document]: From 45be71d9bf2c8cc6619c3c3062d2b37022cebe92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 13:36:35 +0800 Subject: [PATCH 077/135] fixbug: DebugError --- metagpt/roles/qa_engineer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index f950efef4..68138d925 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -112,8 +112,7 @@ class QaEngineer(Role): async def _debug_error(self, msg): run_code_context = RunCodeContext.loads(msg.content) - output_file_repo = CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO) - output_doc = await output_file_repo.get(run_code_context.output_filename) + output_doc = await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).get(run_code_context.output_filename) if not output_doc: return run_code_context.output = output_doc.content From 938fa8a446de3d1fbb50efc780577a1854ec6c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 13:48:25 +0800 Subject: [PATCH 078/135] feat: git archive --- metagpt/software_company.py | 2 ++ metagpt/utils/git_repository.py | 1 + 2 files changed, 3 insertions(+) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index d3c2c463b..5aa0864e0 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -59,4 +59,6 @@ class SoftwareCompany(BaseModel): logger.debug(f"{n_round=}") self._check_balance() await self.environment.run() + if CONFIG.git_repo: + CONFIG.git_repo.archive() return self.environment.history diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index c2eb2360e..660561bf3 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -151,6 +151,7 @@ class GitRepository: :param comments: Comments for the archive commit. """ + logger.info(f"Archive: {[list(self.changed_files.keys())]}") self.add_change(self.changed_files) self.commit(comments) From 8ce6914df21b0799db04a968f3243a591ff14c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 14:04:01 +0800 Subject: [PATCH 079/135] feat: git archive --- metagpt/utils/git_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 660561bf3..ace0cf8a2 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -151,7 +151,7 @@ class GitRepository: :param comments: Comments for the archive commit. """ - logger.info(f"Archive: {[list(self.changed_files.keys())]}") + logger.info(f"Archive: {list(self.changed_files.keys())}") self.add_change(self.changed_files) self.commit(comments) From 882f22da352f8099af6fc0974a292c4866cb6c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 19:56:27 +0800 Subject: [PATCH 080/135] =?UTF-8?q?feat:=20=E6=B5=81=E7=A8=8B=E8=B0=83?= =?UTF-8?q?=E9=80=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/debug_error.py | 45 +++++++++---- metagpt/actions/design_api.py | 93 +++++++++++++-------------- metagpt/actions/project_management.py | 41 +++++++++++- metagpt/actions/run_code.py | 29 +++++---- metagpt/actions/write_code.py | 69 +++++++++++--------- metagpt/actions/write_prd.py | 74 +++++++++++++++++++-- metagpt/const.py | 2 +- metagpt/roles/engineer.py | 8 ++- metagpt/roles/qa_engineer.py | 26 ++++---- metagpt/schema.py | 14 ++++ metagpt/utils/git_repository.py | 9 ++- 11 files changed, 274 insertions(+), 136 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 7a12e18f8..d0c3652b4 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -8,7 +8,10 @@ import re from metagpt.actions.action import Action +from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger +from metagpt.schema import RunCodeResult from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -19,7 +22,20 @@ Based on the message, first, figure out your own role, i.e. Engineer or QaEngine then rewrite the development code or the test code based on your role, the error, and the summary, such that all bugs are fixed and the code performs well. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. The message is as follows: -{context} +# Legacy Code +```python +{code} +``` +--- +# Unit Test Code +```python +{test_code} +``` +--- +# Console logs +```text +{logs} +``` --- Now you should start rewriting the code: ## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. @@ -30,25 +46,26 @@ class DebugError(Action): def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) - # async def run(self, code, error): - # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ - # f"\n\n{error}\n\nPlease try to fix the error in this code." - # fixed_code = await self._aask(prompt) - # return fixed_code - async def run(self, *args, **kwargs) -> str: + output_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get(self.context.output_filename) + if not output_doc: + return "" + output_detail = RunCodeResult.loads(output_doc.content) pattern = r"Ran (\d+) tests in ([\d.]+)s\n\nOK" - matches = re.search(pattern, self.context.output) + matches = re.search(pattern, output_detail.stderr) if matches: - return "", "the original code works fine, no need to debug" + return "" - file_name = self.context.code_filename - logger.info(f"Debug and rewrite {file_name}") - - prompt = PROMPT_TEMPLATE.format(context=self.context.output) + logger.info(f"Debug and rewrite {self.context.code_filename}") + code_doc = await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).get(self.context.code_filename) + if not code_doc: + return "" + test_doc = await CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get(self.context.test_filename) + if not test_doc: + return "" + prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr) rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=rsp) return code diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index a8f89473d..02f87bc47 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -6,7 +6,6 @@ @File : design_api.py """ import json -import shutil from pathlib import Path from typing import List @@ -18,13 +17,11 @@ from metagpt.const import ( SEQ_FLOW_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, SYSTEM_DESIGN_PDF_FILE_REPO, - WORKSPACE_ROOT, ) from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown from metagpt.utils.mermaid import mermaid_to_file templates = { @@ -157,6 +154,34 @@ OUTPUT_MAPPING = { "Anything UNCLEAR": (str, ...), } +MERGE_PROMPT = """ +## Old Design +{old_design} + +## Context +{context} + +----- +Role: You are an architect; The goal is to incrementally update the "Old Design" based on the information provided by the "Context," aiming to design a state-of-the-art (SOTA) Python system compliant with PEP8. Additionally, the objective is to optimize the use of high-quality open-source tools. +Requirement: Fill in the following missing information based on the context, each section name is a key in json +Max Output: 8192 chars or 2048 tokens. Try to use them up. + +## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. + +## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores + +## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here + +## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. + +## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Design" format, +and only output the json inside this tag, nothing else +""" + class WriteDesign(Action): def __init__(self, name, context=None, llm=None): @@ -167,50 +192,6 @@ class WriteDesign(Action): "clearly and in detail." ) - def recreate_workspace(self, workspace: Path): - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # Folder does not exist, but we don't care - workspace.mkdir(parents=True, exist_ok=True) - - async def _save_prd(self, docs_path, resources_path, context): - prd_file = docs_path / "prd.md" - if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: - quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"] - await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") - - if context[-1].instruct_content: - logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict())) - - async def _save_system_design(self, docs_path, resources_path, system_design): - data_api_design = system_design.instruct_content.dict()[ - "Data structures and interface definitions" - ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) - seq_flow = system_design.instruct_content.dict()[ - "Program call flow" - ] # CodeParser.parse_code(block="Program call flow", text=content) - await mermaid_to_file(data_api_design, resources_path / "data_api_design") - await mermaid_to_file(seq_flow, resources_path / "seq_flow") - system_design_file = docs_path / "system_design.md" - logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict()))) - - async def _save(self, context, system_design): - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["Python package name"] - else: - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name - self.recreate_workspace(workspace) - docs_path = workspace / "docs" - resources_path = workspace / "resources" - docs_path.mkdir(parents=True, exist_ok=True) - resources_path.mkdir(parents=True, exist_ok=True) - await self._save_prd(docs_path, resources_path, context) - await self._save_system_design(docs_path, resources_path, system_design) - async def run(self, with_messages, format=CONFIG.prompt_format): # 通过git diff来识别docs/prds下哪些PRD文档发生了变动 prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) @@ -234,7 +215,8 @@ class WriteDesign(Action): filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo ) changed_files.docs[filename] = doc - + if not changed_files.docs: + logger.info("Nothing has changed.") # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=changed_files.json(), instruct_content=changed_files) @@ -253,10 +235,21 @@ class WriteDesign(Action): await self._rename_workspace(system_design) return system_design - async def _merge(self, prd_doc, system_design_doc): + async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): + prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content) + system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) + # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python + # package name" contain space, have to use setattr + setattr( + system_design.instruct_content, + "Python package name", + system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), + ) + system_design_doc.content = system_design.instruct_content.json() return system_design_doc - async def _rename_workspace(self, system_design): + @staticmethod + async def _rename_workspace(system_design): if CONFIG.WORKDIR: # 已经指定了在旧版本上更新 return diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 686aa3689..4fd944027 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -17,6 +17,7 @@ from metagpt.const import ( TASK_PDF_FILE_REPO, WORKSPACE_ROOT, ) +from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template @@ -169,6 +170,35 @@ OUTPUT_MAPPING = { "Anything UNCLEAR": (str, ...), } +MERGE_PROMPT = """ +# Context +{context} + +## Old Tasks +{old_tasks} +----- +Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. +Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them +Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. + +## Required Python third-party packages: Provided in requirements.txt format + +## Required Other language third-party packages: Provided in requirements.txt format + +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + +## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first + +## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first + +## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format, +and only output the json inside this tag, nothing else +""" + class WriteTasks(Action): def __init__(self, name="CreateTasks", context=None, llm=None): @@ -209,6 +239,8 @@ class WriteTasks(Action): ) change_files.docs[filename] = task_doc + if not change_files.docs: + logger.info("Nothing has changed.") # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=change_files.json(), instruct_content=change_files) @@ -216,7 +248,7 @@ class WriteTasks(Action): system_design_doc = await system_design_file_repo.get(filename) task_doc = await tasks_file_repo.get(filename) if task_doc: - task_doc = await self._merge(system_design_doc=system_design_doc, task_dock=task_doc) + task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc) else: rsp = await self._run_new_tasks(context=system_design_doc.content) task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) @@ -234,8 +266,11 @@ class WriteTasks(Action): # self._save(context, rsp) return rsp - async def _merge(self, system_design_doc, task_dock) -> Document: - return task_dock + async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: + prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content) + rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) + task_doc.content = rsp.instruct_content.json() + return task_doc @staticmethod async def _update_requirements(doc): diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index b244577a7..242eaa25d 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -12,6 +12,7 @@ from typing import Tuple from metagpt.actions.action import Action from metagpt.logs import logger +from metagpt.schema import RunCodeResult PROMPT_TEMPLATE = """ Role: You are a senior development and qa engineer, your role is summarize the code running result. @@ -89,14 +90,7 @@ class RunCode(Action): additional_python_paths = [working_directory] + additional_python_paths additional_python_paths = ":".join(additional_python_paths) env["PYTHONPATH"] = additional_python_paths + ":" + env.get("PYTHONPATH", "") - - install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] - logger.info(" ".join(install_command)) - subprocess.run(install_command, check=True, cwd=working_directory, env=env) - - install_pytest_command = ["python", "-m", "pip", "install", "pytest"] - logger.info(" ".join(install_pytest_command)) - subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) + RunCode._install_dependencies(working_directory=working_directory, env=env) # Start the subprocess process = subprocess.Popen( @@ -113,7 +107,7 @@ class RunCode(Action): stdout, stderr = process.communicate() return stdout.decode("utf-8"), stderr.decode("utf-8") - async def run(self, *args, **kwargs) -> str: + async def run(self, *args, **kwargs) -> RunCodeResult: logger.info(f"Running {' '.join(self.context.command)}") if self.context.mode == "script": outs, errs = await self.run_script( @@ -139,7 +133,20 @@ class RunCode(Action): prompt = PROMPT_TEMPLATE.format(context=context) rsp = await self._aask(prompt) + return RunCodeResult(summary=rsp, stdout=outs, stderr=errs) - result = context + rsp + @staticmethod + def _install_dependencies(working_directory, env): + install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] + logger.info(" ".join(install_command)) + try: + subprocess.run(install_command, check=True, cwd=working_directory, env=env) + except subprocess.CalledProcessError as e: + logger.warning(f"{e}") - return result + install_pytest_command = ["python", "-m", "pip", "install", "pytest"] + logger.info(" ".join(install_pytest_command)) + try: + subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) + except subprocess.CalledProcessError as e: + logger.warning(f"{e}") diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index d4d33fe0c..c9b6c3b9e 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -7,16 +7,15 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` value of the `Message` object. """ -import json from tenacity import retry, stop_after_attempt, wait_fixed -from metagpt.actions import WriteDesign from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.const import TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger -from metagpt.schema import CodingContext -from metagpt.utils.common import CodeParser, any_to_str +from metagpt.schema import CodingContext, RunCodeResult +from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ NOTICE @@ -33,8 +32,25 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc 7. Do not use public member functions that do not exist in your design. ----- -# Context -{context} +# Design +```json +{design} +``` +----- +# Tasks +```json +{tasks} +``` +----- +# Legacy Code +```python +{code} +``` +----- +# Debug logs +```text +{logs} +``` ----- ## Format example ----- @@ -51,26 +67,6 @@ class WriteCode(Action): def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) - def _is_invalid(self, filename): - return any(i in filename for i in ["mp3", "wav"]) - - def _save(self, context, filename, code): - # logger.info(filename) - # logger.info(code_rsp) - if self._is_invalid(filename): - return - - design = [i for i in context if i.cause_by == any_to_str(WriteDesign)][0] - - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name - if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): - ws_path = ws_path / ws_name - code_path = ws_path / filename - code_path.parent.mkdir(parents=True, exist_ok=True) - code_path.write_text(code) - logger.info(f"Saving Code to {code_path}") - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) @@ -78,12 +74,21 @@ class WriteCode(Action): return code async def run(self, *args, **kwargs) -> CodingContext: - m = json.loads(self.context.content) - coding_context = CodingContext(**m) - context = "\n".join( - [coding_context.design_doc.content, coding_context.task_doc.content, coding_context.code_doc.content] + coding_context = CodingContext.loads(self.context.content) + test_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get( + "test_" + coding_context.filename + ".json" + ) + logs = "" + if test_doc: + test_detail = RunCodeResult.loads(test_doc.content) + logs = test_detail.stderr + prompt = PROMPT_TEMPLATE.format( + design=coding_context.design_doc.content, + tasks=coding_context.task_doc.content, + code=coding_context.code_doc.content, + logs=logs, + filename=self.context.filename, ) - prompt = PROMPT_TEMPLATE.format(context=context, filename=self.context.filename) logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) coding_context.code_doc.content = code diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 8b03ac29a..532f5bc34 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -219,6 +219,7 @@ There are no unclear points. }, } + OUTPUT_MAPPING = { "Original Requirements": (str, ...), "Product Goals": (List[str], ...), @@ -231,13 +232,60 @@ OUTPUT_MAPPING = { "Anything UNCLEAR": (str, ...), } +IS_RELATIVE_PROMPT = """ +## PRD: +{old_prd} + +## New Requirement: +{requirements} + +___ +You are a professional product manager; You need to assess whether the new requirements are relevant to the existing PRD to determine whether to merge the new requirements into this PRD. +Is the newly added requirement in "New Requirement" related to the PRD? +Respond with `YES` if it is related, `NO` if it is not, and provide the reasons. Return the response in JSON format. +""" + +MERGE_PROMPT = """ +# Context +## Original Requirements +{requirements} + + +## Old PRD +{old_prd} +----- +Role: You are a professional product manager; The goal is to merge the newly added requirements into the existing PRD in order to design a concise, usable, and efficient product. +Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design + +## Original Requirements: Provide as Plain text, place the polished complete original requirements here + +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple + +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less + +## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible + +## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + +## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. + +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower + +## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. +## Anything UNCLEAR: Provide as Plain text. Make clear here. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old PRD" format, +and only output the json inside this tag, nothing else +""" + class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: - # 判断哪些需求文档需要重写：调LLM判断新增需求与prd是否相关，若相关就rewrite prd + # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are + # related to the PRD. If they are related, rewrite the PRD. docs_file_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) requirement_doc = await docs_file_repo.get(REQUIREMENT_FILENAME) prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) @@ -250,14 +298,16 @@ class WritePRD(Action): if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc - # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd + # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: prd_doc = await self._update_prd( requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc - # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 + # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the + # 'publish' message to transition the workflow to the next stage. This design allows room for global + # optimization in subsequent steps. return ActionOutput(content=change_files.json(), instruct_content=change_files) async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: @@ -278,11 +328,23 @@ class WritePRD(Action): prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) return prd - async def _is_relative_to(self, doc1, doc2) -> bool: + async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: + m = json.loads(old_prd_doc.content) + if m.get("Original Requirements") == new_requirement_doc.content: + # There have been no changes in the requirements, so they are considered unrelated. + return False + prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) + res = await self._aask(prompt=prompt) + logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") + if "YES" in res: + return True return False - async def _merge(self, doc1, doc2) -> Document: - pass + async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: + prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) + prd_doc.content = prd.instruct_content.json() + return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: if not prd_doc: diff --git a/metagpt/const.py b/metagpt/const.py index 311712013..49965b622 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -63,4 +63,4 @@ SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" PRD_PDF_FILE_REPO = "resources/prd" TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" TEST_CODES_FILE_REPO = "tests" -OUTPUTS_FILE_REPO = "outputs" +TEST_OUTPUTS_FILE_REPO = "test_outputs" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 89827a1ca..b6ecc4767 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -90,6 +90,8 @@ class Engineer(Role): self._rc.memory.add(msg) changed_files.add(coding_context.code_doc.filename) + if not changed_files: + logger.info("Nothing has changed.") return changed_files async def _act(self) -> Message: @@ -136,8 +138,8 @@ class Engineer(Role): root_path=str(src_file_repo.root_path), filename=task_filename, content=context.json() ) if task_filename in changed_files.docs: - logger.error( - f"Log to expose potential file name conflicts: {coding_doc.json()} & " + logger.warning( + f"Log to expose potential conflicts: {coding_doc.json()} & " f"{changed_files.docs[task_filename].json()}" ) changed_files.docs[task_filename] = coding_doc @@ -168,7 +170,7 @@ class Engineer(Role): old_code_doc = await src_file_repo.get(filename) if not old_code_doc: old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="") - dependencies = {Path(i) for i in dependency.get(old_code_doc.root_relative_path)} + dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)} task_doc = None design_doc = None for i in dependencies: diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 68138d925..a88b01e37 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -9,7 +9,11 @@ """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_NONE, OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO +from metagpt.const import ( + MESSAGE_ROUTE_TO_NONE, + TEST_CODES_FILE_REPO, + TEST_OUTPUTS_FILE_REPO, +) from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext @@ -86,20 +90,17 @@ class QaEngineer(Role): return run_code_context.code = src_doc.content run_code_context.test_code = test_doc.content - result_msg = await RunCode(context=run_code_context, llm=self._llm).run() - run_code_context.output_filename = run_code_context.test_filename + ".md" - await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).save( + result = await RunCode(context=run_code_context, llm=self._llm).run() + run_code_context.output_filename = run_code_context.test_filename + ".json" + await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).save( filename=run_code_context.output_filename, - content=result_msg, + content=result.json(), dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, ) run_code_context.code = None run_code_context.test_code = None - recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself - mappings = { - "Engineer": "Alex", - "QaEngineer": "Edward", - } + recipient = parse_recipient(result.summary) # the recipient might be Engineer or myself + mappings = {"Engineer": "Alex", "QaEngineer": "Edward"} self.publish_message( Message( content=run_code_context.json(), @@ -112,16 +113,11 @@ class QaEngineer(Role): async def _debug_error(self, msg): run_code_context = RunCodeContext.loads(msg.content) - output_doc = await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).get(run_code_context.output_filename) - if not output_doc: - return - run_code_context.output = output_doc.content code = await DebugError(context=run_code_context, llm=self._llm).run() await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).save( filename=run_code_context.code_filename, content=code ) run_code_context.output = None - run_code_context.output_filename = None self.publish_message( Message( content=run_code_context.json(), diff --git a/metagpt/schema.py b/metagpt/schema.py index 53a22f0e6..e910fc866 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -295,3 +295,17 @@ class RunCodeContext(BaseModel): return RunCodeContext(**m) except Exception: return None + + +class RunCodeResult(BaseModel): + summary: str + stdout: str + stderr: str + + @staticmethod + def loads(val: str) -> RunCodeResult | None: + try: + m = json.loads(val) + return RunCodeResult(**m) + except Exception: + return None diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index ace0cf8a2..b8e35199b 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -72,7 +72,14 @@ class GitRepository: :param local_path: The local path where the new Git repository will be initialized. """ - self._repository = Repo.init(path=local_path) + self._repository = Repo.init(path=Path(local_path)) + + gitignore_filename = Path(local_path) / ".gitignore" + ignores = ["__pycache__", "*.pyc"] + with open(str(gitignore_filename), mode="w") as writer: + writer.write("\n".join(ignores)) + self._repository.index.add([".gitignore"]) + self._repository.index.commit("Add .gitignore") def add_change(self, files: Dict): """Add or remove files from the staging area based on the provided changes. From e1cabcad492d48804376a238c13747619396f1cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:21:52 +0800 Subject: [PATCH 081/135] feat: +annotation --- metagpt/actions/debug_error.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index d0c3652b4..7fdc2ef5b 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -4,6 +4,8 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : debug_error.py +@Modified By: mashenquan, 2023/11/27. Divide the context into three components: legacy code, unit test code, and + console log. """ import re From 86c5e5e8e662556204bdc69adf1a050e94962320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:28:07 +0800 Subject: [PATCH 082/135] feat: +annotation --- metagpt/actions/debug_error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 7fdc2ef5b..971f76ca7 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : debug_error.py @Modified By: mashenquan, 2023/11/27. Divide the context into three components: legacy code, unit test code, and - console log. + console log. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. """ import re From 22c5077747b22a4ad7b3bfbe1cd25d867e8c84fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:32:40 +0800 Subject: [PATCH 083/135] feat: +annotation --- metagpt/actions/design_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 02f87bc47..8644aa6a4 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,6 +4,9 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py +@Modified By: mashenquan, 2023/11/27. According to Section 2.2.3.1 of RFC 135, replace file data in the message with + the file name. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration + functionality. """ import json from pathlib import Path From 57d826a40cd0d7bb7a17f522fd6c3099f57bc20d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:44:46 +0800 Subject: [PATCH 084/135] feat: +annotation --- metagpt/actions/design_api.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 8644aa6a4..2f8a306d5 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -196,14 +196,15 @@ class WriteDesign(Action): ) async def run(self, with_messages, format=CONFIG.prompt_format): - # 通过git diff来识别docs/prds下哪些PRD文档发生了变动 + # Use `git diff` to identify which PRD documents have been modified in the `docs/prds` directory. prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) changed_prds = prds_file_repo.changed_files - # 通过git diff来识别docs/system_designs下那些设计文档发生了变动； + # Use `git diff` to identify which design documents in the `docs/system_designs` directory have undergone + # changes. system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_system_designs = system_design_file_repo.changed_files - # 对于那些发生变动的PRD和设计文档，重新生成设计内容； + # For those PRDs and design documents that have undergone changes, regenerate the design content. changed_files = Documents() for filename in changed_prds.keys(): doc = await self._update_system_design( @@ -220,7 +221,8 @@ class WriteDesign(Action): changed_files.docs[filename] = doc if not changed_files.docs: logger.info("Nothing has changed.") - # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 + # Wait until all files under `docs/system_designs/` are processed before sending the publish message, + # leaving room for global optimization in subsequent steps. return ActionOutput(content=changed_files.json(), instruct_content=changed_files) async def _new_system_design(self, context, format=CONFIG.prompt_format): @@ -253,7 +255,7 @@ class WriteDesign(Action): @staticmethod async def _rename_workspace(system_design): - if CONFIG.WORKDIR: # 已经指定了在旧版本上更新 + if CONFIG.WORKDIR: # Updating on the old version has already been specified if it's valid. return if isinstance(system_design, ActionOutput): From 759c8378e42c4b8a76d3d96946325ccd4c5f61d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 11:54:09 +0800 Subject: [PATCH 085/135] feat: +annotation --- metagpt/actions/debug_error.py | 5 +++-- metagpt/actions/design_api.py | 6 +++--- metagpt/actions/project_management.py | 26 +++++--------------------- 3 files changed, 11 insertions(+), 26 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 971f76ca7..e4a15d38d 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -4,8 +4,9 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : debug_error.py -@Modified By: mashenquan, 2023/11/27. Divide the context into three components: legacy code, unit test code, and - console log. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. +@Modified By: mashenquan, 2023/11/27. + 1. Divide the context into three components: legacy code, unit test code, and console log. + 2. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. """ import re diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 2f8a306d5..021edfe72 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,9 +4,9 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py -@Modified By: mashenquan, 2023/11/27. According to Section 2.2.3.1 of RFC 135, replace file data in the message with - the file name. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration - functionality. +@Modified By: mashenquan, 2023/11/27. + 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. + 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. """ import json from pathlib import Path diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 4fd944027..042f1f01c 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -4,6 +4,10 @@ @Time : 2023/5/11 19:12 @Author : alexanderwu @File : project_management.py +@Modified By: mashenquan, 2023/11/27. + 1. Divide the context into three components: legacy code, unit test code, and console log. + 2. Move the document storage operations related to WriteDesign to the save operation of WriteDesign. + 3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality. """ import json from typing import List @@ -11,17 +15,10 @@ from typing import List from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import ( - SYSTEM_DESIGN_FILE_REPO, - TASK_FILE_REPO, - TASK_PDF_FILE_REPO, - WORKSPACE_ROOT, -) +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, TASK_PDF_FILE_REPO from metagpt.logs import logger from metagpt.schema import Document, Documents -from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown templates = { "json": { @@ -204,18 +201,6 @@ class WriteTasks(Action): def __init__(self, name="CreateTasks", context=None, llm=None): super().__init__(name, context, llm) - def _save(self, context, rsp): - if context[-1].instruct_content: - ws_name = context[-1].instruct_content.dict()["Python package name"] - else: - ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / "docs/api_spec_and_tasks.md" - file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) - - # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" - requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) - async def run(self, with_messages, format=CONFIG.prompt_format): system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_system_designs = system_design_file_repo.changed_files @@ -263,7 +248,6 @@ class WriteTasks(Action): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) - # self._save(context, rsp) return rsp async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: From c483d0d7a3c74f8e66ed41d106013a17e80d7d6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 12:00:50 +0800 Subject: [PATCH 086/135] feat: +annotation --- metagpt/actions/project_management.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 042f1f01c..0081fd223 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -208,14 +208,15 @@ class WriteTasks(Action): tasks_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) changed_tasks = tasks_file_repo.changed_files change_files = Documents() - # 根据docs/system_designs/下的git head diff识别哪些task文档需要重写 + # Rewrite the system designs that have undergone changes based on the git head diff under + # `docs/system_designs/`. for filename in changed_system_designs: task_doc = await self._update_tasks( filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo ) change_files.docs[filename] = task_doc - # 根据docs/tasks/下的git head diff识别哪些task文件被用户修改了，需要重写 + # Rewrite the task files that have undergone changes based on the git head diff under docs/tasks/. for filename in changed_tasks: if filename in change_files.docs: continue @@ -226,7 +227,8 @@ class WriteTasks(Action): if not change_files.docs: logger.info("Nothing has changed.") - # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 + # Wait until all files under `docs/tasks/` are processed before sending the publish message, leaving room for + # global optimization in subsequent steps. return ActionOutput(content=change_files.json(), instruct_content=change_files) async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo): From 5ea488d37a2c43df63f48a874b4f83e5bd50e832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 12:03:21 +0800 Subject: [PATCH 087/135] feat: +annotation --- metagpt/actions/project_management.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 0081fd223..ee1632612 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -216,7 +216,7 @@ class WriteTasks(Action): ) change_files.docs[filename] = task_doc - # Rewrite the task files that have undergone changes based on the git head diff under docs/tasks/. + # Rewrite the task files that have undergone changes based on the git head diff under `docs/tasks/`. for filename in changed_tasks: if filename in change_files.docs: continue @@ -227,7 +227,7 @@ class WriteTasks(Action): if not change_files.docs: logger.info("Nothing has changed.") - # Wait until all files under `docs/tasks/` are processed before sending the publish message, leaving room for + # Wait until all files under `docs/tasks/` are processed before sending the publish_message, leaving room for # global optimization in subsequent steps. return ActionOutput(content=change_files.json(), instruct_content=change_files) From a405b4759b64648fe8f59c4ed411955c77db5714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 13:49:15 +0800 Subject: [PATCH 088/135] feat: +annotation --- metagpt/actions/run_code.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 242eaa25d..1e7010e52 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -4,6 +4,14 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : run_code.py +@Modified By: mashenquan, 2023/11/27. + 1. Mark the location of Console logs in the PROMPT_TEMPLATE with markdown code-block formatting to enhance + the understanding for the LLM. + 2. Fix bug: Add the "install dependency" operation. + 3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into + RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. + 4. According to section 2.2.3.5.7 of RFC 135, change the method of transferring file content + (code files, unit test files, log files) from using the message to using the file name. """ import os import subprocess From 0f03645a8920b50b3dcc67817565cdd73f3e0f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:27:56 +0800 Subject: [PATCH 089/135] feat: +annotation --- metagpt/actions/write_code.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index c9b6c3b9e..e9d41bb20 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -6,6 +6,13 @@ @File : write_code.py @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` value of the `Message` object. +@Modified By: mashenquan, 2023-11-27. + 1. Mark the location of Design, Tasks, Legacy Code and Debug logs in the PROMPT_TEMPLATE with markdown + code-block formatting to enhance the understanding for the LLM. + 2. Following the think-act principle, solidify the task parameters when creating the WriteCode object, rather + than passing them in when calling the run function. + 3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into + RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. """ from tenacity import retry, stop_after_attempt, wait_fixed From 9c13958f6c498e9d24fcd951e1c6ced84d35bde9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:31:01 +0800 Subject: [PATCH 090/135] feat: +annotation --- metagpt/actions/write_code_review.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 10e4aec3b..dae1c965f 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -4,6 +4,8 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code_review.py +@Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the + WriteCode object, rather than passing them in when calling the run function. """ from tenacity import retry, stop_after_attempt, wait_fixed From 16226a2e11621b91d05feb4a74e97259f95d66b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:35:22 +0800 Subject: [PATCH 091/135] feat: +annotation --- metagpt/actions/write_prd.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 532f5bc34..68e0e75ba 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -4,6 +4,9 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd.py +@Modified By: mashenquan, 2023/11/27. + 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. + 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. """ from __future__ import annotations From 512e205cd0945be9c6d8c6a980b309b286788557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:38:46 +0800 Subject: [PATCH 092/135] feat: +annotation --- metagpt/actions/project_management.py | 2 +- metagpt/actions/write_prd.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index ee1632612..641d21533 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -6,7 +6,7 @@ @File : project_management.py @Modified By: mashenquan, 2023/11/27. 1. Divide the context into three components: legacy code, unit test code, and console log. - 2. Move the document storage operations related to WriteDesign to the save operation of WriteDesign. + 2. Move the document storage operations related to WritePRD from the save operation of WriteDesign. 3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality. """ import json diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 68e0e75ba..cc21058b4 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023/11/27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. + 3. Move the document storage operations related to WritePRD from the save operation of WriteDesign. """ from __future__ import annotations From fbd24635df779764d9cd5608354ab4b649495f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:40:36 +0800 Subject: [PATCH 093/135] feat: +annotation --- metagpt/actions/write_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 9a9671bab..e980e0831 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -4,6 +4,8 @@ @Time : 2023/5/11 22:12 @Author : alexanderwu @File : environment.py +@Modified By: mashenquan, 2023-11-27. Following the think-act principle, solidify the task parameters when creating the + WriteTest object, rather than passing them in when calling the run function. """ from metagpt.actions.action import Action from metagpt.config import CONFIG From 628ecc0fb7585b749da7d49eb28171395af6b042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:46:25 +0800 Subject: [PATCH 094/135] feat: +annotation --- metagpt/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/config.py b/metagpt/config.py index d059a6a29..a20f58ec1 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -2,6 +2,9 @@ # -*- coding: utf-8 -*- """ Provide configuration, singleton +@Modified By: mashenquan, 2023/11/27. + 1. According to Section 2.2.3.11 of RFC 135, add git repository support. + 2. Add the parameter `src_workspace` for the old version project path. """ import os From 331d74059f18b5fdbb4aedbc8c5ce6a234f7ab4e Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 20 Nov 2023 11:24:46 +0800 Subject: [PATCH 095/135] =?UTF-8?q?1.=20=E5=8A=A8=E4=BD=9C=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20=20=201.=20SummarizeCode=E5=8A=A8=E4=BD=9C=EF=BC=9A?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E5=9F=BA=E4=BA=8E=E4=BB=A3=E7=A0=81=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E6=80=BB=E7=BB=93=EF=BC=8C=E6=80=9D=E8=80=83bug?= =?UTF-8?q?=E3=80=81=E9=80=BB=E8=BE=91=E3=80=81todo=20=20=202.=20CodeRevie?= =?UTF-8?q?w=E5=8A=A8=E4=BD=9C=E4=BC=98=E5=8C=96=EF=BC=9A=E7=9B=AE?= =?UTF-8?q?=E5=89=8D=E5=BC=BA=E5=88=B6=E8=A6=81=E6=B1=82=E5=9B=9E=E7=AD=94?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E6=9C=89=E6=9B=B4=E9=AB=98=E7=9A=84?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E7=8E=87=E4=BA=86=202.=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E7=BB=93=E6=9E=84=20=20=201.=20Document=E7=9A=84=E6=A0=87?= =?UTF-8?q?=E5=87=86=E5=8C=96=EF=BC=9AEnv->Repo->Document=EF=BC=8C?= =?UTF-8?q?=E5=85=B6=E4=B8=ADDocument/Asset/Code=E9=83=BD=E5=8F=AA?= =?UTF-8?q?=E7=94=A8Document=20=20=20=20=201.=20=E5=8E=9F=E7=94=A8?= =?UTF-8?q?=E4=BA=8E=E6=A3=80=E7=B4=A2=E7=9A=84Document=E6=94=B9=E4=B8=BAI?= =?UTF-8?q?ndexableDocument=20=20=202.=20Repo=E7=BB=93=E6=9E=84=E5=BC=95?= =?UTF-8?q?=E5=85=A5=EF=BC=9A=E7=94=A8=E4=BA=8EDocument=E8=A3=85=E8=BD=BD?= =?UTF-8?q?=E4=B8=8E=E5=85=83=E6=95=B0=E6=8D=AE=E8=A3=85=E8=BD=BD=20=20=20?= =?UTF-8?q?3.=20RepoParser=E5=BC=95=E5=85=A5=EF=BC=9A=E5=86=99=E4=BA=86?= =?UTF-8?q?=E4=B8=80=E4=B8=AA=E7=AE=80=E5=8D=95=E7=9A=84AST=20parser?= =?UTF-8?q?=EF=BC=88=E5=90=8E=E7=BB=AD=E5=8F=AF=E8=83=BD=E8=A6=81=E6=8D=A2?= =?UTF-8?q?tree-sitter=EF=BC=89=EF=BC=8C=E7=BB=99=E5=87=BA=E4=BA=86?= =?UTF-8?q?=E6=95=B4=E5=BA=93symbol=203.=20=E9=85=8D=E7=BD=AE=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20=20=201.=20=E9=BB=98=E8=AE=A4=E6=9B=B4=E6=8D=A2?= =?UTF-8?q?=E4=B8=BAgpt-4-1106-preview=EF=BC=8C=E4=BB=A5=E8=8E=B7=E5=BE=97?= =?UTF-8?q?=E6=9C=80=E5=A5=BD=E7=9A=84=E6=95=88=E6=9E=9C=E4=B8=8E=E6=88=90?= =?UTF-8?q?=E6=9C=AC=20=20=202.=20=E6=8F=90=E4=BE=9B~/.metagpt=E4=BD=9C?= =?UTF-8?q?=E4=B8=BA=E9=85=8D=E7=BD=AE=E6=9C=80=E9=AB=98=E4=BC=98=E5=85=88?= =?UTF-8?q?=E7=BA=A7=E7=9B=AE=E5=BD=95=EF=BC=8C=E4=BB=8E=E4=B8=AD=E8=AF=BB?= =?UTF-8?q?=E5=8F=96config.yaml=20=20=203.=20workspace=E5=8F=AF=E4=BB=A5?= =?UTF-8?q?=E7=81=B5=E6=B4=BB=E6=8C=87=E5=AE=9A=E4=BA=86=EF=BC=8C=E5=9C=A8?= =?UTF-8?q?config=E4=B8=AD=E9=85=8D=E7=BD=AE=204.=20metagpt=E4=BD=9C?= =?UTF-8?q?=E4=B8=BA=E9=BB=98=E8=AE=A4=E5=91=BD=E4=BB=A4=E8=A1=8C=EF=BC=8C?= =?UTF-8?q?=E8=80=8C=E9=9D=9Epython=20startup.py=20=20=201.=20=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E6=96=B0=E7=9A=84METAGPT=5FROOT=E7=94=9F=E6=88=90?= =?UTF-8?q?=E6=96=B9=E5=BC=8F=EF=BC=8C=E8=80=8C=E9=9D=9E=E5=AF=BB=E6=89=BE?= =?UTF-8?q?git=EF=BC=8C=E4=BB=A5=E4=BE=BFcli=E5=AE=89=E8=A3=85=20=20=202.?= =?UTF-8?q?=20=E5=91=BD=E4=BB=A4=E8=A1=8C=E7=94=B1fire=E6=8D=A2=E4=B8=BA?= =?UTF-8?q?=E4=BA=86typer=EF=BC=8C=E5=AE=83=E4=BC=9A=E5=B8=A6=E6=9D=A5?= =?UTF-8?q?=E7=9B=B8=E5=AF=B9=E6=9B=B4=E5=A5=BD=E7=9A=84=E4=BD=93=E9=AA=8C?= =?UTF-8?q?=20=20=203.=20project=5Fname=E5=8F=AF=E4=BB=A5=E7=81=B5?= =?UTF-8?q?=E6=B4=BB=E6=8C=87=E5=AE=9A=E4=BA=86=EF=BC=8C=E5=9C=A8metagpt?= =?UTF-8?q?=E5=91=BD=E4=BB=A4=E8=A1=8C=E8=BE=93=E5=85=A5=E4=B8=AD=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=205.=20=E5=85=B6=E4=BB=96=20=20=201.=20BossRequiremen?= =?UTF-8?q?t=20->=20UserRequirement=20=20=202.=20=E5=A4=A7=E9=87=8F?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E6=96=87=E6=9C=AC=E7=9A=84=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=EF=BC=8C=E5=A2=9E=E5=8A=A0=E4=BA=86=E5=8F=AF=E8=AF=BB=E6=80=A7?= =?UTF-8?q?=20=20=203.=20=E4=B8=AD=E9=87=8F=E6=8F=90=E7=A4=BA=E8=AF=8D?= =?UTF-8?q?=E4=BC=98=E5=8C=96=EF=BC=8C=E7=A8=8D=E5=BE=AE=E6=8F=90=E5=8D=87?= =?UTF-8?q?=E4=BA=86=E4=B8=80=E4=BA=9B=E5=87=86=E7=A1=AE=E7=8E=87=20=20=20?= =?UTF-8?q?4.=20=E6=9A=82=E6=97=B6=E5=B1=8F=E8=94=BD=E4=BA=86LongtermMemor?= =?UTF-8?q?y=E7=9B=B8=E5=85=B3=E9=80=BB=E8=BE=91=EF=BC=8C=E8=BF=99?= =?UTF-8?q?=E4=B8=AA=E9=80=BB=E8=BE=91=E5=BA=95=E5=B1=82=E8=B0=83=E7=94=A8?= =?UTF-8?q?=E4=BA=86langchain=E7=9A=84FAISS=EF=BC=8C=E4=BC=9A=E5=B8=A6?= =?UTF-8?q?=E6=9D=A5~5=E7=A7=92=E5=8A=A0=E8=BD=BD=E8=80=97=E6=97=B6=20=20?= =?UTF-8?q?=205.=20=E4=BF=AE=E5=A4=8D=E4=BA=86=E5=AE=89=E8=A3=85=E5=8C=85?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E9=83=A8=E5=88=86=E6=8F=8F=E8=BF=B0=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/agent_creator.py | 7 +- examples/debate.py | 4 +- examples/sk_agent.py | 10 +- metagpt/actions/SummarizeCode.py | 93 ++++++++ metagpt/actions/__init__.py | 4 +- metagpt/actions/add_requirement.py | 4 +- metagpt/actions/design_api.py | 35 ++- metagpt/actions/project_management.py | 13 +- metagpt/actions/write_code.py | 25 ++- metagpt/actions/write_code_review.py | 18 +- metagpt/actions/write_prd.py | 29 +-- metagpt/actions/write_test.py | 2 +- metagpt/config.py | 19 +- metagpt/const.py | 72 +++--- metagpt/document.py | 207 ++++++++++++++++++ metagpt/document_store/base_store.py | 10 +- metagpt/document_store/document.py | 82 ------- metagpt/document_store/faiss_store.py | 10 +- metagpt/document_store/repo_parser.py | 90 ++++++++ metagpt/environment.py | 12 +- metagpt/logs.py | 10 +- metagpt/manager.py | 2 +- metagpt/memory/__init__.py | 4 +- metagpt/roles/engineer.py | 23 +- metagpt/roles/product_manager.py | 4 +- metagpt/roles/qa_engineer.py | 9 +- metagpt/roles/role.py | 5 +- metagpt/roles/sk_agent.py | 4 +- metagpt/software_company.py | 13 -- metagpt/startup.py | 45 ++++ metagpt/team.py | 18 +- metagpt/tools/sd_engine.py | 14 +- metagpt/utils/mermaid.py | 8 +- metagpt/utils/token_counter.py | 7 +- setup.py | 11 +- startup.py | 72 ------ tests/metagpt/actions/mock.py | 4 +- tests/metagpt/actions/test_write_prd.py | 4 +- tests/metagpt/document_store/test_document.py | 16 +- tests/metagpt/memory/test_longterm_memory.py | 12 +- tests/metagpt/memory/test_memory_storage.py | 10 +- tests/metagpt/planner/test_action_planner.py | 4 +- tests/metagpt/planner/test_basic_planner.py | 4 +- tests/metagpt/roles/mock.py | 8 +- tests/metagpt/roles/ui_role.py | 7 +- tests/metagpt/test_environment.py | 4 +- tests/metagpt/tools/test_sd_tool.py | 6 +- tests/metagpt/utils/test_common.py | 6 +- tests/metagpt/utils/test_output_parser.py | 2 +- tests/metagpt/utils/test_read_docx.py | 4 +- 50 files changed, 699 insertions(+), 387 deletions(-) create mode 100644 metagpt/actions/SummarizeCode.py create mode 100644 metagpt/document.py delete mode 100644 metagpt/document_store/document.py create mode 100644 metagpt/document_store/repo_parser.py delete mode 100644 metagpt/software_company.py create mode 100644 metagpt/startup.py delete mode 100644 startup.py diff --git a/examples/agent_creator.py b/examples/agent_creator.py index 325e7c260..bcb9c0c1d 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -5,13 +5,14 @@ Author: garylin2099 ''' import re -from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT +from metagpt.const import METAGPT_ROOT +from metagpt.config import CONFIG from metagpt.actions import Action from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger -with open(PROJECT_ROOT / "examples/build_customized_agent.py", "r") as f: +with open(METAGPT_ROOT / "examples/build_customized_agent.py", "r") as f: # use official example script to guide AgentCreator MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read() @@ -49,7 +50,7 @@ class CreateAgent(Action): pattern = r'```python(.*)```' match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else "" - with open(WORKSPACE_ROOT / "agent_created_agent.py", "w") as f: + with open(CONFIG.workspace_path / "agent_created_agent.py", "w") as f: f.write(code_text) return code_text diff --git a/examples/debate.py b/examples/debate.py index a37e60848..0f5d1591b 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -8,7 +8,7 @@ import platform import fire from metagpt.team import Team -from metagpt.actions import Action, BossRequirement +from metagpt.actions import Action, UserRequirement from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger @@ -49,7 +49,7 @@ class Debator(Role): ): super().__init__(name, profile, **kwargs) self._init_actions([SpeakAloud]) - self._watch([BossRequirement, SpeakAloud]) + self._watch([UserRequirement, SpeakAloud]) self.name = name self.opponent_name = opponent_name diff --git a/examples/sk_agent.py b/examples/sk_agent.py index a7513e838..647ea4380 100644 --- a/examples/sk_agent.py +++ b/examples/sk_agent.py @@ -13,7 +13,7 @@ from semantic_kernel.planning import SequentialPlanner # from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -39,7 +39,7 @@ async def basic_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def sequential_planner_example(): @@ -53,7 +53,7 @@ async def sequential_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def basic_planner_web_search_example(): @@ -64,7 +64,7 @@ async def basic_planner_web_search_example(): role.import_skill(SkSearchEngine(), "WebSearchSkill") # role.import_semantic_skill_from_directory(skills_directory, "QASkill") - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def action_planner_example(): @@ -75,7 +75,7 @@ async def action_planner_example(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - await role.run(Message(content=task, cause_by=BossRequirement)) # it will choose mathskill.Add + await role.run(Message(content=task, cause_by=UserRequirement)) # it will choose mathskill.Add if __name__ == "__main__": diff --git a/metagpt/actions/SummarizeCode.py b/metagpt/actions/SummarizeCode.py new file mode 100644 index 000000000..1015d3bfb --- /dev/null +++ b/metagpt/actions/SummarizeCode.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Author : alexanderwu +@File : SummarizeCode.py +""" + +from metagpt.actions.action import Action +from metagpt.logs import logger +from metagpt.schema import Message +from metagpt.utils.common import CodeParser +from tenacity import retry, stop_after_attempt, wait_fixed + +PROMPT_TEMPLATE = """ +NOTICE +Role: You are a professional software engineer, and your main task is to review the code. +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". + +----- +# Context +{context} +----- + +## Code Review All: 请你对历史所有文件进行阅读，分析每个文件是否都完整实现了用户需求，找到可能的bug，如函数未实现、调用错误、未引用等 + +## Summary: 根据历史文件的实现情况进行总结 + +## Call flow: 根据实现的函数，使用mermaid绘制完整的调用链 + +## TODOs: 这里写出需要修改的文件列表，我们会在之后进行修改 + +""" + +FORMAT_EXAMPLE = """ + +## Code Review All + +### a.py +- 它少实现了xxx需求... +- 字段yyy没有给出... +- ... + +### b.py +... + +### c.py +... + +## Call flow +```mermaid +flowchart TB + c1-->a2 + subgraph one + a1-->a2 + end + subgraph two + b1-->b2 + end + subgraph three + c1-->c2 + end +``` + +## Summary +- a.py:... +- b.py:... +- c.py:... +- ... + +## TODOs +1. ... +2. ... +3. ... + +""" + + +class SummarizeCode(Action): + def __init__(self, name="SummaryCode", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) + + @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) + async def write_code_review_all(self, prompt): + code_rsp = await self._aask(prompt) + return code_rsp + + async def run(self, context): + format_example = FORMAT_EXAMPLE.format() + prompt = PROMPT_TEMPLATE.format(context=context, format_example=format_example) + logger.info(f'Code review all..') + rsp = await self.write_code_review_all(prompt) + return rsp + \ No newline at end of file diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index b004bd58e..79ff94b3e 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -9,7 +9,7 @@ from enum import Enum from metagpt.actions.action import Action from metagpt.actions.action_output import ActionOutput -from metagpt.actions.add_requirement import BossRequirement +from metagpt.actions.add_requirement import UserRequirement from metagpt.actions.debug_error import DebugError from metagpt.actions.design_api import WriteDesign from metagpt.actions.design_api_review import DesignReview @@ -28,7 +28,7 @@ from metagpt.actions.write_test import WriteTest class ActionType(Enum): """All types of Actions, used for indexing.""" - ADD_REQUIREMENT = BossRequirement + ADD_REQUIREMENT = UserRequirement WRITE_PRD = WritePRD WRITE_PRD_REVIEW = WritePRDReview WRITE_DESIGN = WriteDesign diff --git a/metagpt/actions/add_requirement.py b/metagpt/actions/add_requirement.py index 7dc09d062..8e2c56a62 100644 --- a/metagpt/actions/add_requirement.py +++ b/metagpt/actions/add_requirement.py @@ -8,7 +8,7 @@ from metagpt.actions import Action -class BossRequirement(Action): - """Boss Requirement without any implementation details""" +class UserRequirement(Action): + """User Requirement without any implementation details""" async def run(self, *args, **kwargs): raise NotImplementedError diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 75df8b909..f58d49495 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,7 +11,6 @@ from typing import List from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template @@ -27,21 +26,20 @@ templates = { ## Format example {format_example} ----- -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system Requirement: Fill in the following missing information based on the context, each section name is a key in json -Max Output: 8192 chars or 2048 tokens. Try to use them up. -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. +## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -52,7 +50,7 @@ and only output the json inside this tag, nothing else "Implementation approach": "We will ...", "Python package name": "snake_game", "File list": ["main.py"], - "Data structures and interface definitions": ' + "Data structures and interfaces": ' classDiagram class Game{ +int score @@ -81,20 +79,19 @@ and only output the json inside this tag, nothing else ----- Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -Max Output: 8192 chars or 2048 tokens. Try to use them up. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """, "FORMAT_EXAMPLE": """ @@ -114,7 +111,7 @@ We will ... ] ``` -## Data structures and interface definitions +## Data structures and interfaces ```mermaid classDiagram class Game{ @@ -143,7 +140,7 @@ OUTPUT_MAPPING = { "Implementation approach": (str, ...), "Python package name": (str, ...), "File list": (List[str], ...), - "Data structures and interface definitions": (str, ...), + "Data structures and interfaces": (str, ...), "Program call flow": (str, ...), "Anything UNCLEAR": (str, ...), } @@ -177,8 +174,8 @@ class WriteDesign(Action): async def _save_system_design(self, docs_path, resources_path, system_design): data_api_design = system_design.instruct_content.dict()[ - "Data structures and interface definitions" - ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) + "Data structures and interfaces" + ] # CodeParser.parse_code(block="Data structures and interfaces", text=content) seq_flow = system_design.instruct_content.dict()[ "Program call flow" ] # CodeParser.parse_code(block="Program call flow", text=content) @@ -193,7 +190,7 @@ class WriteDesign(Action): ws_name = system_design.instruct_content.dict()["Python package name"] else: ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name + workspace = CONFIG.workspace_path / ws_name self.recreate_workspace(workspace) docs_path = workspace / "docs" resources_path = workspace / "resources" diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index b395fa64e..467cb4d83 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -9,7 +9,6 @@ from typing import List from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template from metagpt.utils.json_to_markdown import json_to_markdown @@ -27,9 +26,9 @@ Role: You are a project manager; the goal is to break down tasks according to PR Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. -## Required Python third-party packages: Provided in requirements.txt format +## Required Python third-party packages: Provide Python list[str] in requirements.txt format -## Required Other language third-party packages: Provided in requirements.txt format +## Required Other language third-party packages: Provide Python list[str] in requirements.txt format ## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. @@ -39,7 +38,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -95,7 +94,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. """, "FORMAT_EXAMPLE": ''' @@ -171,11 +170,11 @@ class WriteTasks(Action): ws_name = context[-1].instruct_content.dict()["Python package name"] else: ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / "docs/api_spec_and_tasks.md" + file_path = CONFIG.workspace_path / ws_name / "docs/api_spec_and_tasks.md" file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" + requirements_path = CONFIG.workspace_path / ws_name / "requirements.txt" requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) async def run(self, context, format=CONFIG.prompt_format): diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index c000805c5..176718dfc 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -7,7 +7,7 @@ """ from metagpt.actions import WriteDesign from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser @@ -18,19 +18,22 @@ NOTICE Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code: {filename} Write code with triple quoto, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN. -5. Think before writing: What should be implemented and provided in this document? -6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. -7. Do not use public member functions that do not exist in your design. - ----- # Context {context} ----- + +## Code: {filename} Write code with triple quoto, based on the following list and context. +1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. +2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets +3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. +5. Think before writing: What should be implemented and provided in this document? +6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +7. Do not use public member functions that do not exist in your design. +8. Before using a variable, make sure you reference it first +9. Write out EVERY DETAIL, DON'T LEAVE TODO. + ## Format example ----- ## Code: {filename} @@ -58,7 +61,7 @@ class WriteCode(Action): design = [i for i in context if i.cause_by == WriteDesign][0] ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name + ws_path = CONFIG.workspace_path / ws_name if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): ws_path = ws_path / ws_name code_path = ws_path / filename diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 4ff4d6cf6..c6538bf7b 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -17,16 +17,14 @@ NOTICE Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code Review: Based on the following context and code, and following the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. -``` -1. Check 0: Is the code implemented as per the requirements? -2. Check 1: Are there any issues with the code logic? -3. Check 2: Does the existing code follow the "Data structures and interface definitions"? -4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented? -5. Check 4: Does the code have unnecessary or lack dependencies? -``` +## Code Review: Based on the following context and code, follow the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. +1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. +2. Are there any issues with the code logic? If so, how to solve it? +3. Does the existing code follow the "Data structures and interfaces"? +4. Is there a function in the code that is not fully implemented? If so, how to implement it? +5. Does the code have unnecessary or lack dependencies? If so, how to solve it? -## Rewrite Code: {filename} Base on "Code Review" and the source code, rewrite code with triple quotes. Do your utmost to optimize THIS SINGLE FILE. +## Rewrite Code: rewrite {filename} based on "Code Review" with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. ----- # Context {context} @@ -47,7 +45,7 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc FORMAT_EXAMPLE = """ ## Code Review -1. The code ... +1. No, we should add the logic of ... 2. ... 3. ... 4. ... diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index bd04ca79e..584d31998 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -46,24 +46,25 @@ quadrantChart {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design +Requirements: According to the context, fill in the following missing information, each section name is a key in json ## Original Requirements: Provide as Plain text, place the polished complete original requirements here -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible +## Competitive Analysis: Provided as Python list[str], up to 8 competitive product analyses ## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. +## Requirement Analysis: Provide as Plain text. -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards ## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. + +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -131,30 +132,30 @@ quadrantChart {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design +Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. ## Original Requirements: Provide as Plain text, place the polished complete original requirements here -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories ## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible ## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. +## Requirement Analysis: Provide as Plain text. -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards ## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """, "FORMAT_EXAMPLE": """ --- ## Original Requirements -The boss ... +The user ... ## Product Goals ```python diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 35ff36dc2..2f4988c09 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -15,7 +15,7 @@ NOTICE 2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases. 3. Attention1: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script. 4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. +5. Attention3: YOU MUST FOLLOW "Data structures and interfaces". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. 6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail? 7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. diff --git a/metagpt/config.py b/metagpt/config.py index 3f9e742bd..1a9cdb4d2 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -8,7 +8,9 @@ import os import openai import yaml -from metagpt.const import PROJECT_ROOT +from pathlib import Path + +from metagpt.const import METAGPT_ROOT, DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.singleton import Singleton @@ -35,13 +37,14 @@ class Config(metaclass=Singleton): """ _instance = None - key_yaml_file = PROJECT_ROOT / "config/key.yaml" - default_yaml_file = PROJECT_ROOT / "config/config.yaml" + home_yaml_file = Path.home() / ".metagpt/config.yaml" + key_yaml_file = METAGPT_ROOT / "config/key.yaml" + default_yaml_file = METAGPT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): self._configs = {} self._init_with_config_files_and_env(self._configs, yaml_file) - logger.info("Config loading done.") + # logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") @@ -94,12 +97,18 @@ class Config(metaclass=Singleton): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) + self._ensure_workspace_exists() + + def _ensure_workspace_exists(self): + self.workspace_path.mkdir(parents=True, exist_ok=True) + logger.info(f"WORKSPACE_PATH set to {self.workspace_path}") def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" configs.update(os.environ) - for _yaml_file in [yaml_file, self.key_yaml_file]: + for _yaml_file in [yaml_file, self.key_yaml_file, self.home_yaml_file]: if not _yaml_file.exists(): continue diff --git a/metagpt/const.py b/metagpt/const.py index 407ce803a..14e692487 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -5,44 +5,54 @@ @Author : alexanderwu @File : const.py """ +import os from pathlib import Path from loguru import logger - -def get_project_root(): - """Search upwards to find the project root directory.""" - current_path = Path.cwd() - while True: - if ( - (current_path / ".git").exists() - or (current_path / ".project_root").exists() - or (current_path / ".gitignore").exists() - ): - # use metagpt with git clone will land here - logger.info(f"PROJECT_ROOT set to {str(current_path)}") - return current_path - parent_path = current_path.parent - if parent_path == current_path: - # use metagpt with pip install will land here - cwd = Path.cwd() - logger.info(f"PROJECT_ROOT set to current working directory: {str(cwd)}") - return cwd - current_path = parent_path +import metagpt -PROJECT_ROOT = get_project_root() -DATA_PATH = PROJECT_ROOT / "data" -WORKSPACE_ROOT = PROJECT_ROOT / "workspace" -PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" -UT_PATH = PROJECT_ROOT / "data/ut" -SWAGGER_PATH = UT_PATH / "files/api/" -UT_PY_PATH = UT_PATH / "files/ut/" -API_QUESTIONS_PATH = UT_PATH / "files/question/" -YAPI_URL = "http://yapi.deepwisdomai.com/" -TMP = PROJECT_ROOT / "tmp" +def get_metagpt_package_root(): + """Get the root directory of the installed package.""" + package_root = Path(metagpt.__file__).parent.parent + logger.info(f"Package root set to {str(package_root)}") + return package_root + + +def get_metagpt_root(): + """Get the project root directory.""" + # Check if a project root is specified in the environment variable + project_root_env = os.getenv('METAGPT_PROJECT_ROOT') + if project_root_env: + project_root = Path(project_root_env) + logger.info(f"PROJECT_ROOT set from environment variable to {str(project_root)}") + else: + # Fallback to package root if no environment variable is set + project_root = get_metagpt_package_root() + return project_root + + +# METAGPT PROJECT ROOT AND VARS + +METAGPT_ROOT = get_metagpt_root() +DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace" + +DATA_PATH = METAGPT_ROOT / "data" RESEARCH_PATH = DATA_PATH / "research" TUTORIAL_PATH = DATA_PATH / "tutorial_docx" INVOICE_OCR_TABLE_PATH = DATA_PATH / "invoice_table" +UT_PATH = DATA_PATH / "ut" +SWAGGER_PATH = UT_PATH / "files/api/" +UT_PY_PATH = UT_PATH / "files/ut/" +API_QUESTIONS_PATH = UT_PATH / "files/question/" -SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills" +TMP = METAGPT_ROOT / "tmp" + +SOURCE_ROOT = METAGPT_ROOT / "metagpt" +PROMPT_PATH = SOURCE_ROOT / "prompts" +SKILL_DIRECTORY = SOURCE_ROOT / "skills" + + +# REAL CONSTS MEM_TTL = 24 * 30 * 3600 +YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/document.py b/metagpt/document.py new file mode 100644 index 000000000..044210218 --- /dev/null +++ b/metagpt/document.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/6/8 14:03 +@Author : alexanderwu +@File : document.py +""" + +from typing import Union, Optional +from pathlib import Path +from pydantic import BaseModel, Field +import pandas as pd +from langchain.document_loaders import ( + TextLoader, + UnstructuredPDFLoader, + UnstructuredWordDocumentLoader, +) +from langchain.text_splitter import CharacterTextSplitter +from tqdm import tqdm + +from metagpt.logs import logger + + +def validate_cols(content_col: str, df: pd.DataFrame): + if content_col not in df.columns: + raise ValueError("Content column not found in DataFrame.") + + +def read_data(data_path: Path): + suffix = data_path.suffix + if '.xlsx' == suffix: + data = pd.read_excel(data_path) + elif '.csv' == suffix: + data = pd.read_csv(data_path) + elif '.json' == suffix: + data = pd.read_json(data_path) + elif suffix in ('.docx', '.doc'): + data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() + elif '.txt' == suffix: + data = TextLoader(str(data_path)).load() + text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) + texts = text_splitter.split_documents(data) + data = texts + elif '.pdf' == suffix: + data = UnstructuredPDFLoader(str(data_path), mode="elements").load() + else: + raise NotImplementedError("File format not supported.") + return data + + +class Document(BaseModel): + """ + Document: Handles operations related to document files. + """ + content: str = Field(default='') + file_path: Path = Field(default=None) + + @classmethod + def from_path(cls, file_path: Path): + """ + Create a Document instance from a file path. + """ + if not file_path.exists(): + raise FileNotFoundError(f"File {file_path} not found.") + content = file_path.read_text() + return cls(content=content, file_path=file_path) + + @classmethod + def from_text(cls, text: str, file_path: Optional[Path] = None): + """ + Create a Document from a text string. + """ + return cls(content=text, file_path=file_path) + + def to_path(self, file_path: Optional[Path] = None): + """ + Save content to the specified file path. + """ + if file_path is not None: + self.file_path = file_path + + if self.file_path is None: + raise ValueError("File path is not set.") + + self.file_path.parent.mkdir(parents=True, exist_ok=True) + self.file_path.write_text(self.content) + + def persist(self): + """ + Persist document to disk. + """ + return self.to_path() + + +class IndexableDocument(Document): + """ + Advanced document handling: For vector databases or search engines. + """ + data: Union[pd.DataFrame, list] + content_col: Optional[str] = Field(default='') + meta_col: Optional[str] = Field(default='') + + class Config: + arbitrary_types_allowed = True + + @classmethod + def from_path(cls, data_path: Path, content_col='content', meta_col='metadata'): + if not data_path.exists(): + raise FileNotFoundError(f"File {data_path} not found.") + data = read_data(data_path) + content = data_path.read_text() + if isinstance(data, pd.DataFrame): + validate_cols(content_col, data) + return cls(data=data, content=content, content_col=content_col, meta_col=meta_col) + + def _get_docs_and_metadatas_by_df(self) -> (list, list): + df = self.data + docs = [] + metadatas = [] + for i in tqdm(range(len(df))): + docs.append(df[self.content_col].iloc[i]) + if self.meta_col: + metadatas.append({self.meta_col: df[self.meta_col].iloc[i]}) + else: + metadatas.append({}) + return docs, metadatas + + def _get_docs_and_metadatas_by_langchain(self) -> (list, list): + data = self.data + docs = [i.page_content for i in data] + metadatas = [i.metadata for i in data] + return docs, metadatas + + def get_docs_and_metadatas(self) -> (list, list): + if isinstance(self.data, pd.DataFrame): + return self._get_docs_and_metadatas_by_df() + elif isinstance(self.data, list): + return self._get_docs_and_metadatas_by_langchain() + else: + raise NotImplementedError("Data type not supported for metadata extraction.") + + +class Repo(BaseModel): + + # Name of this repo. + name: str = Field(default="") + docs: dict[Path, Document] = Field(default_factory=dict) + codes: dict[Path, Document] = Field(default_factory=dict) + assets: dict[Path, Document] = Field(default_factory=dict) + repo_path: Path = Field(default_factory=Path) + + def _path(self, filename): + return self.repo_path / filename + + @classmethod + def from_path(cls, repo_path: Path): + """Load documents, code, and assets from a repository path.""" + repo_path.mkdir(parents=True, exist_ok=True) + repo = Repo(repo_path = repo_path) + for file_path in repo_path.rglob('*'): + if file_path.is_file(): + repo._set(file_path.read_text(), file_path) + return repo + + def to_path(self): + """Persist all documents, code, and assets to the given repository path.""" + for doc in self.docs.values(): + doc.to_path() + for code in self.codes.values(): + code.to_path() + for asset in self.assets.values(): + asset.to_path() + + def _set(self, content: str, file_path: Path): + """Add a document to the appropriate category based on its file extension.""" + file_ext = file_path.suffix + + doc = Document(content=content, file_path=file_path) + if file_ext.lower() == '.md': + self.docs[file_path] = doc + elif file_ext.lower() in ['.py', '.js', '.css', '.html']: + self.codes[file_path] = doc + else: + self.assets[file_path] = doc + return doc + + def set(self, content: str, filename: str): + """Set a document and persist it to disk.""" + file_path = self._path(filename) + doc = self._set(content, file_path) + doc.to_path() + + def get(self, filename: str) -> Optional[Document]: + """Get a document by its filename.""" + path = self._path(filename) + return self.docs.get(path) or self.codes.get(path) or self.assets.get(path) + + +def main(): + repo1 = Repo.from_path(Path("/Users/alexanderwu/workspace/t1")) + repo1.set("wtf content", "doc/wtf_file.md") + repo1.set("wtf code", "code/wtf_file.py") + logger.info(repo1) # check doc + + +if __name__ == '__main__': + main() diff --git a/metagpt/document_store/base_store.py b/metagpt/document_store/base_store.py index 5d7015e8b..84b47a98c 100644 --- a/metagpt/document_store/base_store.py +++ b/metagpt/document_store/base_store.py @@ -28,20 +28,20 @@ class BaseStore(ABC): class LocalStore(BaseStore, ABC): - def __init__(self, raw_data: Path, cache_dir: Path = None): - if not raw_data: + def __init__(self, raw_data_path: Path, cache_dir: Path = None): + if not raw_data_path: raise FileNotFoundError self.config = Config() - self.raw_data = raw_data + self.raw_data_path = raw_data_path if not cache_dir: - cache_dir = raw_data.parent + cache_dir = raw_data_path.parent self.cache_dir = cache_dir self.store = self._load() if not self.store: self.store = self.write() def _get_index_and_store_fname(self): - fname = self.raw_data.name.split('.')[0] + fname = self.raw_data_path.name.split('.')[0] index_file = self.cache_dir / f"{fname}.index" store_file = self.cache_dir / f"{fname}.pkl" return index_file, store_file diff --git a/metagpt/document_store/document.py b/metagpt/document_store/document.py deleted file mode 100644 index e4b9473c7..000000000 --- a/metagpt/document_store/document.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/6/8 14:03 -@Author : alexanderwu -@File : document.py -""" -from pathlib import Path - -import pandas as pd -from langchain.document_loaders import ( - TextLoader, - UnstructuredPDFLoader, - UnstructuredWordDocumentLoader, -) -from langchain.text_splitter import CharacterTextSplitter -from tqdm import tqdm - - -def validate_cols(content_col: str, df: pd.DataFrame): - if content_col not in df.columns: - raise ValueError - - -def read_data(data_path: Path): - suffix = data_path.suffix - if '.xlsx' == suffix: - data = pd.read_excel(data_path) - elif '.csv' == suffix: - data = pd.read_csv(data_path) - elif '.json' == suffix: - data = pd.read_json(data_path) - elif suffix in ('.docx', '.doc'): - data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() - elif '.txt' == suffix: - data = TextLoader(str(data_path)).load() - text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) - texts = text_splitter.split_documents(data) - data = texts - elif '.pdf' == suffix: - data = UnstructuredPDFLoader(str(data_path), mode="elements").load() - else: - raise NotImplementedError - return data - - -class Document: - - def __init__(self, data_path, content_col='content', meta_col='metadata'): - self.data = read_data(data_path) - if isinstance(self.data, pd.DataFrame): - validate_cols(content_col, self.data) - self.content_col = content_col - self.meta_col = meta_col - - def _get_docs_and_metadatas_by_df(self) -> (list, list): - df = self.data - docs = [] - metadatas = [] - for i in tqdm(range(len(df))): - docs.append(df[self.content_col].iloc[i]) - if self.meta_col: - metadatas.append({self.meta_col: df[self.meta_col].iloc[i]}) - else: - metadatas.append({}) - - return docs, metadatas - - def _get_docs_and_metadatas_by_langchain(self) -> (list, list): - data = self.data - docs = [i.page_content for i in data] - metadatas = [i.metadata for i in data] - return docs, metadatas - - def get_docs_and_metadatas(self) -> (list, list): - if isinstance(self.data, pd.DataFrame): - return self._get_docs_and_metadatas_by_df() - elif isinstance(self.data, list): - return self._get_docs_and_metadatas_by_langchain() - else: - raise NotImplementedError - \ No newline at end of file diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index dd450010d..885ad3e15 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -15,15 +15,15 @@ from langchain.vectorstores import FAISS from metagpt.const import DATA_PATH from metagpt.document_store.base_store import LocalStore -from metagpt.document_store.document import Document +from metagpt.document import IndexableDocument from metagpt.logs import logger class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col='source', content_col='output'): + def __init__(self, raw_data_path: Path, cache_dir=None, meta_col='source', content_col='output'): self.meta_col = meta_col self.content_col = content_col - super().__init__(raw_data, cache_dir) + super().__init__(raw_data_path, cache_dir) def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() @@ -60,9 +60,9 @@ class FaissStore(LocalStore): def write(self): """Initialize the index and library based on the Document (JSON / XLSX, etc.) file provided by the user.""" - if not self.raw_data.exists(): + if not self.raw_data_path.exists(): raise FileNotFoundError - doc = Document(self.raw_data, self.content_col, self.meta_col) + doc = IndexableDocument.from_path(self.raw_data_path, self.content_col, self.meta_col) docs, metadatas = doc.get_docs_and_metadatas() self.store = self._write(docs, metadatas) diff --git a/metagpt/document_store/repo_parser.py b/metagpt/document_store/repo_parser.py new file mode 100644 index 000000000..f7e2b0f4a --- /dev/null +++ b/metagpt/document_store/repo_parser.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/17 17:58 +@Author : alexanderwu +@File : repo_parser.py +""" +import json +import pathlib +import ast + +import pandas as pd + + +class RepoParser: + def __init__(self): + self.base_directory = None + + def parse_file(self, file_path): + """Parse a Python file in the repository.""" + try: + return ast.parse(file_path.read_text()).body + except: + return [] + + def extract_class_and_function_info(self, tree, file_path): + """Extract class, function, and global variable information from the AST.""" + file_info = { + "file": str(file_path.relative_to(self.base_directory)), + "classes": [], + "functions": [], + "globals": [] + } + + for node in tree: + if isinstance(node, ast.ClassDef): + class_methods = [m.name for m in node.body if is_func(m)] + file_info["classes"].append({"name": node.name, "methods": class_methods}) + elif is_func(node): + file_info["functions"].append(node.name) + elif isinstance(node, ast.Assign) or isinstance(node, ast.AnnAssign): + for target in node.targets if isinstance(node, ast.Assign) else [node.target]: + if isinstance(target, ast.Name): + file_info["globals"].append(target.id) + return file_info + + def generate_json_structure(self, directory, output_path): + """Generate a JSON file documenting the repository structure.""" + files_classes = [] + for path in directory.rglob('*.py'): + tree = self.parse_file(path) + file_info = self.extract_class_and_function_info(tree, path) + files_classes.append(file_info) + + output_path.write_text(json.dumps(files_classes, indent=4)) + + def generate_dataframe_structure(self, directory, output_path): + """Generate a DataFrame documenting the repository structure and save as CSV.""" + files_classes = [] + for path in directory.rglob('*.py'): + tree = self.parse_file(path) + file_info = self.extract_class_and_function_info(tree, path) + files_classes.append(file_info) + + df = pd.DataFrame(files_classes) + df.to_csv(output_path, index=False) + + def generate_structure(self, directory_path, output_path=None, mode='json'): + """Generate the structure of the repository as a specified format.""" + self.base_directory = pathlib.Path(directory_path) + output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}" + output_path = pathlib.Path(output_path) if output_path else output_file + + if mode == 'json': + self.generate_json_structure(self.base_directory, output_path) + elif mode == 'csv': + self.generate_dataframe_structure(self.base_directory, output_path) + + +def is_func(node): + return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + + +def main(): + repo_parser = RepoParser() + repo_parser.generate_structure("/Users/alexanderwu/git/mg1/metagpt", "/Users/alexanderwu/git/mg1/mg1-structure.csv", mode='csv') + + +if __name__ == '__main__': + main() diff --git a/metagpt/environment.py b/metagpt/environment.py index 24e6ada2f..38077c90d 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -10,20 +10,22 @@ from typing import Iterable from pydantic import BaseModel, Field +# from metagpt.document import Document +from metagpt.document import Repo from metagpt.memory import Memory from metagpt.roles import Role from metagpt.schema import Message class Environment(BaseModel): - """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 - Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - + """ + Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles """ roles: dict[str, Role] = Field(default_factory=dict) memory: Memory = Field(default_factory=Memory) history: str = Field(default='') + repo: Repo = Field(default_factory=Repo) class Config: arbitrary_types_allowed = True @@ -50,6 +52,10 @@ class Environment(BaseModel): self.memory.add(message) self.history += f"\n{message}" + def publish_doc(self, content: str, filename: str): + """向当前环境发布文档（包括代码）""" + self.repo.set(content, filename) + async def run(self, k=1): """处理一次所有信息的运行 Process all Role runs at once diff --git a/metagpt/logs.py b/metagpt/logs.py index b2052e9b8..afebbfed9 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -10,15 +10,15 @@ import sys from loguru import logger as _logger -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT + def define_log_level(print_level="INFO", logfile_level="DEBUG"): - """调整日志级别到level之上 - Adjust the log level to above level - """ + """Adjust the log level to above level""" _logger.remove() _logger.add(sys.stderr, level=print_level) - _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) + _logger.add(METAGPT_ROOT / 'logs/log.txt', level=logfile_level) return _logger + logger = define_log_level() diff --git a/metagpt/manager.py b/metagpt/manager.py index 9d238c621..7cbbe651e 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -14,7 +14,7 @@ class Manager: def __init__(self, llm: LLM = LLM()): self.llm = llm # Large Language Model self.role_directions = { - "BOSS": "Product Manager", + "User": "Product Manager", "Product Manager": "Architect", "Architect": "Engineer", "Engineer": "QA Engineer", diff --git a/metagpt/memory/__init__.py b/metagpt/memory/__init__.py index 710930626..bd6e72163 100644 --- a/metagpt/memory/__init__.py +++ b/metagpt/memory/__init__.py @@ -7,10 +7,10 @@ """ from metagpt.memory.memory import Memory -from metagpt.memory.longterm_memory import LongTermMemory +# from metagpt.memory.longterm_memory import LongTermMemory __all__ = [ "Memory", - "LongTermMemory", + # "LongTermMemory", ] diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 1f6685b38..171af47f0 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -11,7 +11,8 @@ from collections import OrderedDict from pathlib import Path from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.SummarizeCode import SummarizeCode +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -80,13 +81,13 @@ class Engineer(Role): self.n_borg = n_borg @classmethod - def parse_tasks(self, task_msg: Message) -> list[str]: + def parse_tasks(cls, task_msg: Message) -> list[str]: if task_msg.instruct_content: return task_msg.instruct_content.dict().get("Task list") return CodeParser.parse_file_list(block="Task list", text=task_msg.content) @classmethod - def parse_code(self, code_text: str) -> str: + def parse_code(cls, code_text: str) -> str: return CodeParser.parse_code(block="", text=code_text) @classmethod @@ -98,10 +99,10 @@ class Engineer(Role): def get_workspace(self) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace_path / "src" workspace = self.parse_workspace(msg) # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace_path / workspace / workspace def recreate_workspace(self): workspace = self.get_workspace() @@ -167,7 +168,7 @@ class Engineer(Role): ) return msg - async def _act_sp_precision(self) -> Message: + async def _act_sp_with_cr(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later for todo in self.todos: """ @@ -191,7 +192,6 @@ class Engineer(Role): code = rewrite_code except Exception as e: logger.error("code review failed!", e) - pass file_path = self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) @@ -199,6 +199,13 @@ class Engineer(Role): code_msg = todo + FILENAME_CODE_SEP + str(file_path) code_msg_all.append(code_msg) + context = [] + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + for m in msg: + context.append(m.content) + context_str = "\n".join(context) + code_review_all = await SummarizeCode().run(context=context_str) + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" @@ -209,5 +216,5 @@ class Engineer(Role): """Determines the mode of action based on whether code review is used.""" logger.info(f"{self._setting}: ready to WriteCode") if self.use_code_review: - return await self._act_sp_precision() + return await self._act_sp_with_cr() return await self._act_sp() diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index a58ea5385..f6172b607 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : product_manager.py """ -from metagpt.actions import BossRequirement, WritePRD +from metagpt.actions import UserRequirement, WritePRD from metagpt.roles import Role @@ -38,4 +38,4 @@ class ProductManager(Role): """ super().__init__(name, profile, goal, constraints) self._init_actions([WritePRD]) - self._watch([BossRequirement]) + self._watch([UserRequirement]) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index a763c2ce8..f124646b3 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -16,7 +16,8 @@ from metagpt.actions import ( WriteDesign, WriteTest, ) -from metagpt.const import WORKSPACE_ROOT +# from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -50,13 +51,13 @@ class QaEngineer(Role): def get_workspace(self, return_proj_dir=True) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace_path / "src" workspace = self.parse_workspace(msg) # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. if return_proj_dir: - return WORKSPACE_ROOT / workspace + return CONFIG.workspace_path / workspace # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace_path / workspace / workspace def write_file(self, filename: str, code: str): workspace = self.get_workspace() / "tests" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b96c361c0..d772c0748 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -17,7 +17,8 @@ from metagpt.config import CONFIG from metagpt.actions import Action, ActionOutput from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory +from metagpt.memory import Memory +# from metagpt.memory import LongTermMemory from metagpt.schema import Message PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -78,7 +79,7 @@ class RoleContext(BaseModel): """Role Runtime Context""" env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) - long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) + # long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index b27841d74..4069f4836 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -9,7 +9,7 @@ from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner from semantic_kernel.planning.basic_planner import BasicPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.actions.execute_task import ExecuteTask from metagpt.logs import logger from metagpt.roles import Role @@ -39,7 +39,7 @@ class SkAgent(Role): """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self._init_actions([ExecuteTask()]) - self._watch([BossRequirement]) + self._watch([UserRequirement]) self.kernel = make_sk_kernel() # how funny the interface is inconsistent diff --git a/metagpt/software_company.py b/metagpt/software_company.py deleted file mode 100644 index d44a0068a..000000000 --- a/metagpt/software_company.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/12 00:30 -@Author : alexanderwu -@File : software_company.py -""" -from metagpt.team import Team as SoftwareCompany - -import warnings -warnings.warn("metagpt.software_company is deprecated and will be removed in the future" - "Please use metagpt.team instead. SoftwareCompany class is now named as Team.", - DeprecationWarning, 2) diff --git a/metagpt/startup.py b/metagpt/startup.py new file mode 100644 index 000000000..d8ca4072f --- /dev/null +++ b/metagpt/startup.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from pathlib import Path +import asyncio +import typer + +app = typer.Typer() + + +@app.command() +def startup( + idea: str = typer.Argument(..., help="Your innovative idea, such as 'Create a 2048 game.'"), + investment: float = typer.Option(3.0, help="Dollar amount to invest in the AI company."), + n_round: int = typer.Option(5, help="Number of rounds for the simulation."), + code_review: bool = typer.Option(True, help="Whether to use code review."), + run_tests: bool = typer.Option(False, help="Whether to enable QA for adding & running tests."), + implement: bool = typer.Option(True, help="Enable or disable code implementation."), + project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'"), +): + """Run a startup. Be a boss.""" + from metagpt.roles import ProductManager, Architect, ProjectManager, Engineer, QaEngineer + from metagpt.team import Team + + company = Team() + company.hire( + [ + ProductManager(), + Architect(), + ProjectManager(), + ] + ) + + if implement or code_review: + company.hire([Engineer(n_borg=5, use_code_review=code_review)]) + + if run_tests: + company.hire([QaEngineer()]) + + company.invest(investment) + company.start_project(project_name, idea) + asyncio.run(company.run(n_round=n_round)) + + +if __name__ == "__main__": + app() diff --git a/metagpt/team.py b/metagpt/team.py index 67d3ecec8..2332aaa46 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -7,7 +7,7 @@ """ from pydantic import BaseModel, Field -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.config import CONFIG from metagpt.environment import Environment from metagpt.logs import logger @@ -21,7 +21,7 @@ class Team(BaseModel): Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a platform for instant messaging, dedicated to perform any multi-agent activity, such as collaboratively writing executable code. """ - environment: Environment = Field(default_factory=Environment) + env: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") @@ -30,7 +30,7 @@ class Team(BaseModel): def hire(self, roles: list[Role]): """Hire roles to cooperate""" - self.environment.add_roles(roles) + self.env.add_roles(roles) def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" @@ -42,10 +42,12 @@ class Team(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea, send_to: str = ""): - """Start a project from publishing boss requirement.""" + def start_project(self, project_name, idea, send_to: str = ""): + """Start a project from publishing user requirement.""" self.idea = idea - self.environment.publish_message(Message(role="Human", content=idea, cause_by=BossRequirement, send_to=send_to)) + # If user set project_name, then use it. + self.env.repo.name = project_name + self.env.publish_message(Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to)) def _save(self): logger.info(self.json()) @@ -57,6 +59,6 @@ class Team(BaseModel): n_round -= 1 logger.debug(f"{n_round=}") self._check_balance() - await self.environment.run() - return self.environment.history + await self.env.run() + return self.env.history \ No newline at end of file diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index 1d9cd0b2a..4f010a912 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -13,12 +13,10 @@ from typing import List from aiohttp import ClientSession from PIL import Image, PngImagePlugin -from metagpt.config import Config -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG +# from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -config = Config() - payload = { "prompt": "", "negative_prompt": "(easynegative:0.8),black, dark,Low resolution", @@ -56,9 +54,8 @@ default_negative_prompt = "(easynegative:0.8),black, dark,Low resolution" class SDEngine: def __init__(self): # Initialize the SDEngine with configuration - self.config = Config() - self.sd_url = self.config.get("SD_URL") - self.sd_t2i_url = f"{self.sd_url}{self.config.get('SD_T2I_API')}" + self.sd_url = CONFIG.get("SD_URL") + self.sd_t2i_url = f"{self.sd_url}{CONFIG.get('SD_T2I_API')}" # Define default payload settings for SD API self.payload = payload logger.info(self.sd_t2i_url) @@ -81,7 +78,7 @@ class SDEngine: return self.payload def _save(self, imgs, save_name=""): - save_dir = WORKSPACE_ROOT / "resources" / "SD_Output" + save_dir = CONFIG.workspace_path / "resources" / "SD_Output" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) @@ -125,6 +122,7 @@ def batch_decode_base64_to_image(imgs, save_dir="", save_name=""): save_name = join(save_dir, save_name) decode_base64_to_image(_img, save_name=save_name) + if __name__ == "__main__": engine = SDEngine() prompt = "pixel style, game design, a game interface should be minimalistic and intuitive with the score and high score displayed at the top. The snake and its food should be easily distinguishable. The game should have a simple color scheme, with a contrasting color for the snake and its food. Complete interface boundary" diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 204c22c67..eb85a3f90 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -10,7 +10,7 @@ import os from pathlib import Path from metagpt.config import CONFIG -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists @@ -69,7 +69,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, if stdout: logger.info(stdout.decode()) if stderr: - logger.error(stderr.decode()) + logger.warning(stderr.decode()) else: if engine == "playwright": from metagpt.utils.mmdc_playwright import mermaid_to_file @@ -141,6 +141,6 @@ MMC2 = """sequenceDiagram if __name__ == "__main__": loop = asyncio.new_event_loop() - result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) - result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) loop.close() diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 1af96f272..33bcd01a5 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -21,6 +21,7 @@ TOKEN_COSTS = { "gpt-4-32k": {"prompt": 0.06, "completion": 0.12}, "gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12}, "gpt-4-0613": {"prompt": 0.06, "completion": 0.12}, + "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069} # 32k version, prompt + completion tokens=0.005￥/k-tokens } @@ -37,6 +38,7 @@ TOKEN_MAX = { "gpt-4-32k": 32768, "gpt-4-32k-0314": 32768, "gpt-4-0613": 8192, + "gpt-4-1106-preview": 128000, "text-embedding-ada-002": 8192, "chatglm_turbo": 32768 } @@ -56,16 +58,17 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613", + "gpt-4-1106-preview", }: tokens_per_message = 3 tokens_per_name = 1 elif model == "gpt-3.5-turbo-0301": tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n tokens_per_name = -1 # if there's a name, the role is omitted - elif "gpt-3.5-turbo" in model: + elif "gpt-3.5-turbo" == model: print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.") return count_message_tokens(messages, model="gpt-3.5-turbo-0613") - elif "gpt-4" in model: + elif "gpt-4" == model: print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.") return count_message_tokens(messages, model="gpt-4-0613") else: diff --git a/setup.py b/setup.py index 239156ae3..e7462767f 100644 --- a/setup.py +++ b/setup.py @@ -31,14 +31,14 @@ with open(path.join(here, "requirements.txt"), encoding="utf-8") as f: setup( name="metagpt", version="0.3.0", - description="The Multi-Role Meta Programming Framework", + description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/geekan/MetaGPT", author="Alexander Wu", author_email="alexanderwu@fuzhi.ai", - license="Apache 2.0", - keywords="metagpt multi-role multi-agent programming gpt llm", + license="MIT", + keywords="metagpt multi-role multi-agent programming gpt llm metaprogramming", packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, @@ -52,4 +52,9 @@ setup( cmdclass={ "install_mermaid": InstallMermaidCLI, }, + entry_points={ + 'console_scripts': [ + 'metagpt=metagpt.startup:app', + ], + }, ) diff --git a/startup.py b/startup.py deleted file mode 100644 index e9fbf94d3..000000000 --- a/startup.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import asyncio - -import fire - -from metagpt.roles import ( - Architect, - Engineer, - ProductManager, - ProjectManager, - QaEngineer, -) -from metagpt.team import Team - - -async def startup( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = False, - run_tests: bool = False, - implement: bool = True, -): - """Run a startup. Be a boss.""" - company = Team() - company.hire( - [ - ProductManager(), - Architect(), - ProjectManager(), - ] - ) - - # if implement or code_review - if implement or code_review: - # developing features: implement the idea - company.hire([Engineer(n_borg=5, use_code_review=code_review)]) - - if run_tests: - # developing features: run tests on the spot and identify bugs - # (bug fixing capability comes soon!) - company.hire([QaEngineer()]) - - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) - - -def main( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = True, - run_tests: bool = False, - implement: bool = True, -): - """ - We are a software startup comprised of AI. By investing in us, - you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea, such as "Creating a snake game." - :param investment: As an investor, you have the opportunity to contribute - a certain dollar amount to this AI company. - :param n_round: - :param code_review: Whether to use code review. - :return: - """ - asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) - - -if __name__ == "__main__": - fire.Fire(main) diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index a800690e8..5be1d8001 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -100,7 +100,7 @@ For testing, we can use the PyTest framework. This is a mature full-featured Pyt file_list = ["main.py", "room.py", "player.py", "game.py", "object.py", "puzzle.py", "test_game.py"] ``` -## Data structures and interface definitions: +## Data structures and interfaces: ```mermaid classDiagram class Room{ @@ -209,7 +209,7 @@ Shared knowledge for this project includes understanding the basic principles of """ ``` -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. ```python """ The original requirements did not specify whether the game should have a save/load feature, multiplayer support, or any specific graphical user interface. More information on these aspects could help in further refining the product design and requirements. diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 38e4e5221..18675ecc3 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -7,7 +7,7 @@ """ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.logs import logger from metagpt.roles.product_manager import ProductManager from metagpt.schema import Message @@ -17,7 +17,7 @@ from metagpt.schema import Message async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" - prd = await product_manager.handle(Message(content=requirements, cause_by=BossRequirement)) + prd = await product_manager.handle(Message(content=requirements, cause_by=UserRequirement)) logger.info(requirements) logger.info(prd) diff --git a/tests/metagpt/document_store/test_document.py b/tests/metagpt/document_store/test_document.py index 5ae357fb1..13c0921a3 100644 --- a/tests/metagpt/document_store/test_document.py +++ b/tests/metagpt/document_store/test_document.py @@ -7,22 +7,22 @@ """ import pytest -from metagpt.const import DATA_PATH -from metagpt.document_store.document import Document +from metagpt.const import METAGPT_ROOT +from metagpt.document import IndexableDocument CASES = [ - ("st/faq.xlsx", "Question", "Answer", 1), - ("cases/faq.csv", "Question", "Answer", 1), + ("requirements.txt", None, None, 0), + # ("cases/faq.csv", "Question", "Answer", 1), # ("cases/faq.json", "Question", "Answer", 1), - ("docx/faq.docx", None, None, 1), - ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 - ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 + # ("docx/faq.docx", None, None, 1), + # ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 + # ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 ] @pytest.mark.parametrize("relative_path, content_col, meta_col, threshold", CASES) def test_document(relative_path, content_col, meta_col, threshold): - doc = Document(DATA_PATH / relative_path, content_col, meta_col) + doc = IndexableDocument.from_path(METAGPT_ROOT / relative_path, content_col, meta_col) rsp = doc.get_docs_and_metadatas() assert len(rsp[0]) > threshold assert len(rsp[1]) > threshold diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index dc5540520..ac9362937 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -4,7 +4,7 @@ from metagpt.config import CONFIG from metagpt.schema import Message -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.roles.role import RoleContext from metagpt.memory import LongTermMemory @@ -15,24 +15,24 @@ def test_ltm_search(): assert len(openai_api_key) > 20 role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(watch=[BossRequirement]) + rc = RoleContext(watch=[UserRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) idea = 'Write a cli snake game' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + message = Message(role='User', content=idea, cause_by=UserRequirement) news = ltm.find_news([message]) assert len(news) == 1 ltm.add(message) sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_message = Message(role='User', content=sim_idea, cause_by=UserRequirement) news = ltm.find_news([sim_message]) assert len(news) == 0 ltm.add(sim_message) new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) news = ltm.find_news([new_message]) assert len(news) == 1 ltm.add(new_message) @@ -48,7 +48,7 @@ def test_ltm_search(): assert len(news) == 0 new_idea = 'Write a Battle City' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) news = ltm_new.find_news([new_message]) assert len(news) == 1 diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index 6bb3e8f1d..bd4441641 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -6,7 +6,7 @@ from typing import List from metagpt.memory.memory_storage import MemoryStorage from metagpt.schema import Message -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.actions import WritePRD from metagpt.actions.action_output import ActionOutput @@ -14,7 +14,7 @@ from metagpt.actions.action_output import ActionOutput def test_idea_message(): idea = 'Write a cli snake game' role_id = 'UTUser1(Product Manager)' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + message = Message(role='User', content=idea, cause_by=UserRequirement) memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -24,12 +24,12 @@ def test_idea_message(): assert memory_storage.is_initialized is True sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_message = Message(role='User', content=sim_idea, cause_by=UserRequirement) new_messages = memory_storage.search(sim_message) assert len(new_messages) == 0 # similar, return [] new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content @@ -49,7 +49,7 @@ def test_actionout_message(): ic_obj = ActionOutput.create_model_class('prd', out_mapping) role_id = 'UTUser2(Architect)' - content = 'The boss has requested the creation of a command-line interface (CLI) snake game' + content = 'The user has requested the creation of a command-line interface (CLI) snake game' message = Message(content=content, instruct_content=ic_obj(**out_data), role='user', diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index 5ab9a493f..8efe6cfc4 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -9,7 +9,7 @@ import pytest from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -23,7 +23,7 @@ async def test_action_planner(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - role.recv(Message(content=task, cause_by=BossRequirement)) + role.recv(Message(content=task, cause_by=UserRequirement)) await role._think() # it will choose mathskill.Add assert "1100" == (await role._act()).content diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index 03a82ec5e..f6d44ba03 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -8,7 +8,7 @@ import pytest from semantic_kernel.core_skills import TextSkill -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -26,7 +26,7 @@ async def test_basic_planner(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - role.recv(Message(content=task, cause_by=BossRequirement)) + role.recv(Message(content=task, cause_by=UserRequirement)) await role._think() # assuming sk_agent will think he needs WriterSkill.Brainstorm and WriterSkill.Translate assert "WriterSkill.Brainstorm" in role.plan.generated_plan.result diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 52fc4a3c1..fbad06acb 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -5,10 +5,10 @@ @Author : alexanderwu @File : mock.py """ -from metagpt.actions import BossRequirement, WriteDesign, WritePRD, WriteTasks +from metagpt.actions import UserRequirement, WriteDesign, WritePRD, WriteTasks from metagpt.schema import Message -BOSS_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" +USER_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" DETAIL_REQUIREMENT = """需求：开发一个基于LLM（大语言模型）与私有知识库的搜索引擎，希望有几点能力 1. 用户可以在私有知识库进行搜索，再根据大语言模型进行总结，输出的结果包括了总结 @@ -94,7 +94,7 @@ SYSTEM_DESIGN = '''## Python package name ] ``` -## Data structures and interface definitions +## Data structures and interfaces ```mermaid classDiagram class Main { @@ -252,7 +252,7 @@ a = 'a' class MockMessages: - req = Message(role="Boss", content=BOSS_REQUIREMENT, cause_by=BossRequirement) + req = Message(role="User", content=USER_REQUIREMENT, cause_by=UserRequirement) prd = Message(role="Product Manager", content=PRD, cause_by=WritePRD) system_design = Message(role="Architect", content=SYSTEM_DESIGN, cause_by=WriteDesign) tasks = Message(role="Project Manager", content=TASKS, cause_by=WriteTasks) diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index a45a89cde..102c6ebd6 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -8,7 +8,8 @@ from functools import wraps from importlib import import_module from metagpt.actions import Action, ActionOutput, WritePRD -from metagpt.const import WORKSPACE_ROOT +# from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -29,7 +30,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Selected Elements:Provide as Plain text, up to 5 specified elements, clear and simple ## HTML Layout:Provide as Plain text, use standard HTML code ## CSS Styles (styles.css):Provide as Plain text,use standard css code -## Anything UNCLEAR:Provide as Plain text. Make clear here. +## Anything UNCLEAR:Provide as Plain text. Try to clarify it. """ @@ -214,7 +215,7 @@ class UIDesign(Action): logger.info("Finish icon design using StableDiffusion API") async def _save(self, css_content, html_content): - save_dir = WORKSPACE_ROOT / "resources" / "codes" + save_dir = CONFIG.workspace_path / "resources" / "codes" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) # Save CSS and HTML content to files diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index a0f1f6257..b27bc3da7 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -8,7 +8,7 @@ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.environment import Environment from metagpt.logs import logger from metagpt.manager import Manager @@ -49,7 +49,7 @@ async def test_publish_and_process_message(env: Environment): env.add_roles([product_manager, architect]) env.set_manager(Manager()) - env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) + env.publish_message(Message(role="User", content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement)) await env.run(k=2) logger.info(f"{env.history=}") diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 77e53c7dc..fea58bc29 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -4,7 +4,9 @@ # import os -from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.tools.sd_engine import SDEngine + def test_sd_engine_init(): @@ -21,5 +23,5 @@ def test_sd_engine_generate_prompt(): async def test_sd_engine_run_t2i(): sd_engine = SDEngine() await sd_engine.run_t2i(prompts=["test"]) - img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png" + img_path = CONFIG.workspace_path / "resources" / "SD_Output" / "output_0.png" assert os.path.exists(img_path) == True diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index ec4443175..b6c000f9b 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -10,7 +10,7 @@ import os import pytest -from metagpt.const import get_project_root +from metagpt.const import get_metagpt_root class TestGetProjectRoot: @@ -20,11 +20,11 @@ class TestGetProjectRoot: os.chdir(abs_root) def test_get_project_root(self): - project_root = get_project_root() + project_root = get_metagpt_root() assert project_root.name == 'metagpt' def test_get_root_exception(self): with pytest.raises(Exception) as exc_info: self.change_etc_dir() - get_project_root() + get_metagpt_root() assert str(exc_info.value) == "Project root not found." diff --git a/tests/metagpt/utils/test_output_parser.py b/tests/metagpt/utils/test_output_parser.py index 4e362f9f7..99ab1f79e 100644 --- a/tests/metagpt/utils/test_output_parser.py +++ b/tests/metagpt/utils/test_output_parser.py @@ -218,7 +218,7 @@ We need clarification on how the high score should be stored. Should it persist } t_text1 = '''## Original Requirements: -The boss wants to create a web-based version of the game "Fly Bird". +The user wants to create a web-based version of the game "Fly Bird". ## Product Goals: diff --git a/tests/metagpt/utils/test_read_docx.py b/tests/metagpt/utils/test_read_docx.py index a7d0774a8..adf473ae7 100644 --- a/tests/metagpt/utils/test_read_docx.py +++ b/tests/metagpt/utils/test_read_docx.py @@ -6,12 +6,12 @@ @File : test_read_docx.py """ -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.utils.read_document import read_docx class TestReadDocx: def test_read_docx(self): - docx_sample = PROJECT_ROOT / "tests/data/docx_for_test.docx" + docx_sample = METAGPT_ROOT / "tests/data/docx_for_test.docx" docx = read_docx(docx_sample) assert len(docx) == 6 From 715a1d874aa1d690aa9f7c5b27d404e6d9c1b19a Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 27 Nov 2023 15:48:07 +0800 Subject: [PATCH 096/135] fix config --- config/config.yaml | 6 +++--- metagpt/actions/SummarizeCode.py | 14 ++++++-------- metagpt/actions/write_code.py | 2 +- requirements.txt | 3 ++- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index bed67083c..9acdbe8a1 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,9 +7,9 @@ ## Or, you can configure OPENAI_PROXY to access official OPENAI_API_BASE. OPENAI_API_BASE: "https://api.openai.com/v1" #OPENAI_PROXY: "http://127.0.0.1:8118" -#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model -OPENAI_API_MODEL: "gpt-4" -MAX_TOKENS: 1500 +#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model +OPENAI_API_MODEL: "gpt-4-1106-preview" +MAX_TOKENS: 4096 RPM: 10 #### if Spark diff --git a/metagpt/actions/SummarizeCode.py b/metagpt/actions/SummarizeCode.py index 1015d3bfb..49a350b75 100644 --- a/metagpt/actions/SummarizeCode.py +++ b/metagpt/actions/SummarizeCode.py @@ -5,11 +5,10 @@ @File : SummarizeCode.py """ +from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import Message -from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE @@ -23,10 +22,10 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc ## Code Review All: 请你对历史所有文件进行阅读，分析每个文件是否都完整实现了用户需求，找到可能的bug，如函数未实现、调用错误、未引用等 -## Summary: 根据历史文件的实现情况进行总结 - ## Call flow: 根据实现的函数，使用mermaid绘制完整的调用链 +## Summary: 根据历史文件的实现情况进行总结 + ## TODOs: 这里写出需要修改的文件列表，我们会在之后进行修改 """ @@ -80,14 +79,13 @@ class SummarizeCode(Action): super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def write_code_review_all(self, prompt): + async def summarize_code(self, prompt): code_rsp = await self._aask(prompt) return code_rsp async def run(self, context): format_example = FORMAT_EXAMPLE.format() prompt = PROMPT_TEMPLATE.format(context=context, format_example=format_example) - logger.info(f'Code review all..') - rsp = await self.write_code_review_all(prompt) + logger.info("Code review all..") + rsp = await self.summarize_code(prompt) return rsp - \ No newline at end of file diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 176718dfc..1f6d16b3b 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -5,13 +5,13 @@ @Author : alexanderwu @File : write_code.py """ +from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions import WriteDesign from metagpt.actions.action import Action from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE diff --git a/requirements.txt b/requirements.txt index f0169d7fa..f233e398f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,8 @@ channels==4.0.0 # docx==0.2.4 #faiss==1.5.3 faiss_cpu==1.7.4 -fire==0.4.0 +# fire==0.4.0 +typer # godot==0.1.1 # google_api_python_client==2.93.0 lancedb==0.1.16 From 22288a342dcbb029447e6d896148bf22da4e9da3 Mon Sep 17 00:00:00 2001 From: geekan Date: Mon, 27 Nov 2023 15:36:50 +0800 Subject: [PATCH 097/135] =?UTF-8?q?1.=20=E5=8A=A8=E4=BD=9C=E4=BC=98?= =?UTF-8?q?=E5=8C=96=20=20=201.=20SummarizeCode=E5=8A=A8=E4=BD=9C=EF=BC=9A?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E5=9F=BA=E4=BA=8E=E4=BB=A3=E7=A0=81=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E6=80=BB=E7=BB=93=EF=BC=8C=E6=80=9D=E8=80=83bug?= =?UTF-8?q?=E3=80=81=E9=80=BB=E8=BE=91=E3=80=81todo=20=20=202.=20CodeRevie?= =?UTF-8?q?w=E5=8A=A8=E4=BD=9C=E4=BC=98=E5=8C=96=EF=BC=9A=E7=9B=AE?= =?UTF-8?q?=E5=89=8D=E5=BC=BA=E5=88=B6=E8=A6=81=E6=B1=82=E5=9B=9E=E7=AD=94?= =?UTF-8?q?=E9=97=AE=E9=A2=98=EF=BC=8C=E6=9C=89=E6=9B=B4=E9=AB=98=E7=9A=84?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E7=8E=87=E4=BA=86=20=20=20=20=201.=20?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86LGTM/LBTM=E7=9A=84=E5=9B=9E=E7=AD=94?= =?UTF-8?q?=EF=BC=8C=E5=9C=A8LGTM=E6=97=B6=E4=BC=9A=E5=8F=8A=E6=97=B6?= =?UTF-8?q?=E5=81=9C=E6=AD=A2=EF=BC=8C=E4=B8=8D=E9=87=8D=E5=86=99=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=20=20=20=20=202.=20=E7=9B=AE=E5=89=8D=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E4=BA=86=E8=AE=BE=E7=BD=AE=E4=B8=AD=E7=9A=84=E5=8F=82?= =?UTF-8?q?=E6=95=B0code=5Freview=5Fk=5Ftimes=EF=BC=8C=E4=B8=8Ereflexion?= =?UTF-8?q?=E7=B1=BB=E4=BC=BC=EF=BC=8C=E8=AE=BE=E7=BD=AE=E4=B8=BA2=20=20?= =?UTF-8?q?=20=20=203.=20=E4=BB=8D=E7=84=B6=E6=9C=89=E6=A6=82=E7=8E=87?= =?UTF-8?q?=E5=8F=91=E7=94=9F=E6=8C=87=E4=BB=A4=E4=B8=8D=E9=81=B5=E5=BE=AA?= =?UTF-8?q?=EF=BC=8C=E5=B0=A4=E5=85=B6=E6=98=AF=E4=BC=9A=E6=9C=89=E6=AF=94?= =?UTF-8?q?=E8=BE=83=E9=AB=98=E7=9A=84=E6=A6=82=E7=8E=87=E5=8F=91=E7=94=9F?= =?UTF-8?q?=E5=90=8C=E6=97=B6review=E5=A4=9A=E4=B8=AA=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=96=87=E4=BB=B6=EF=BC=8C=E8=BF=98=E6=B2=A1=E6=83=B3=E5=A5=BD?= =?UTF-8?q?=E6=80=8E=E4=B9=88=E8=A7=A3=E5=86=B3=20#FIXME=20=20=203.=20?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86env=E5=88=B0Action=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E4=B8=AD=EF=BC=8C=E7=8E=B0=E5=9C=A8=E5=8F=AF=E4=BB=A5=E7=9B=B4?= =?UTF-8?q?=E6=8E=A5=E8=B0=83=E7=94=A8=E7=8E=AF=E5=A2=83=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E4=BA=86=20=20=204.=20WriteDesign=EF=BC=9A=E5=8E=BB=E9=99=A4?= =?UTF-8?q?=E4=BA=86=E5=AF=B9project=5Fname=E7=9A=84=E7=BA=A0=E6=AD=A3?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=EF=BC=8C=E7=8E=B0=E5=9C=A8=E5=BC=95=E5=AF=BC?= =?UTF-8?q?=E4=B8=8B=E5=8F=AF=E4=BB=A5=E4=B8=80=E6=AC=A1=E7=94=9F=E6=88=90?= =?UTF-8?q?=E5=AF=B9=20=20=20=20=201.=20=E4=BF=AE=E6=94=B9=E4=BA=86?= =?UTF-8?q?=E6=8F=90=E7=A4=BA=E8=AF=8D=E4=B8=AD=E7=9A=84##=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=EF=BC=8C=E6=94=B9=E4=B8=BA=E4=BA=86JSON=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=202.=20=E6=95=B0=E6=8D=AE=E7=BB=93=E6=9E=84=20=20=201?= =?UTF-8?q?.=20Document=E7=9A=84=E6=A0=87=E5=87=86=E5=8C=96=EF=BC=9AEnv->R?= =?UTF-8?q?epo->Document=EF=BC=8C=E5=85=B6=E4=B8=ADDocument/Asset/Code?= =?UTF-8?q?=E9=83=BD=E6=98=AFDocument=20=20=20=20=201.=20=E5=8E=9F?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E6=A3=80=E7=B4=A2=E7=9A=84Document=E6=94=B9?= =?UTF-8?q?=E4=B8=BAIndexableDocument=20=20=202.=20Repo=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E5=BC=95=E5=85=A5=EF=BC=9A=E7=94=A8=E4=BA=8EDocument=E8=A3=85?= =?UTF-8?q?=E8=BD=BD=E4=B8=8E=E5=85=83=E6=95=B0=E6=8D=AE=E8=A3=85=E8=BD=BD?= =?UTF-8?q?=20=20=203.=20RepoParser=E5=BC=95=E5=85=A5=EF=BC=9A=E5=86=99?= =?UTF-8?q?=E4=BA=86=E4=B8=80=E4=B8=AA=E7=AE=80=E5=8D=95=E7=9A=84AST=20par?= =?UTF-8?q?ser=EF=BC=88=E5=90=8E=E7=BB=AD=E5=8F=AF=E8=83=BD=E8=A6=81?= =?UTF-8?q?=E6=8D=A2tree-sitter=EF=BC=89=EF=BC=8C=E7=BB=99=E5=87=BA?= =?UTF-8?q?=E4=BA=86=E6=95=B4=E5=BA=93symbol=20=20=204.=20Env=E4=B8=AD?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86set/get/set=5Fdoc/get=5Fdoc=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E7=94=A8=E4=BA=8Eset/get=E5=8D=95=E4=B8=AA?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E6=88=96=E8=80=85=E4=B8=80=E4=B8=AADocument?= =?UTF-8?q?=E3=80=82=E8=BF=99=E4=B8=AA=E9=80=BB=E8=BE=91=E5=90=8E=E7=BB=AD?= =?UTF-8?q?=E6=88=96=E8=AE=B8=E4=BC=9A=E8=BF=9B=E4=B8=80=E6=AD=A5=E7=AE=80?= =?UTF-8?q?=E5=8C=96=203.=20=E9=85=8D=E7=BD=AE=E4=BC=98=E5=8C=96=20=20=201?= =?UTF-8?q?.=20=E9=BB=98=E8=AE=A4=E6=9B=B4=E6=8D=A2=E4=B8=BAgpt-4-1106-pre?= =?UTF-8?q?view=EF=BC=8C=E4=BB=A5=E8=8E=B7=E5=BE=97=E6=9C=80=E5=A5=BD?= =?UTF-8?q?=E7=9A=84=E6=95=88=E6=9E=9C=E4=B8=8E=E6=88=90=E6=9C=AC=20=20=20?= =?UTF-8?q?2.=20=E6=8F=90=E4=BE=9B~/.metagpt=E4=BD=9C=E4=B8=BA=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=9C=80=E9=AB=98=E4=BC=98=E5=85=88=E7=BA=A7=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=EF=BC=8C=E4=BB=8E=E4=B8=AD=E8=AF=BB=E5=8F=96config.ya?= =?UTF-8?q?ml=20=20=203.=20workspace=E5=8F=AF=E4=BB=A5=E7=81=B5=E6=B4=BB?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E4=BA=86=EF=BC=8C=E5=9C=A8config=E4=B8=AD?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=20=20=204.=20project=5Fname=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E7=94=B1=E5=91=BD=E4=BB=A4=E8=A1=8C=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E4=B8=94=E6=94=B9=E4=B8=BA=E7=94=B1ProductMa?= =?UTF-8?q?nager=E7=94=9F=E6=88=90=204.=20metagpt=E4=BD=9C=E4=B8=BA?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=91=BD=E4=BB=A4=E8=A1=8C=EF=BC=8C=E8=80=8C?= =?UTF-8?q?=E9=9D=9Epython=20startup.py=20metagpt=20--help?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit metagpt --project-name game_2048 "make a 2048 game" metagpt "make a 2048 game" metagpt --project-name game_2048 --inc "将2048改为4096" metagpt --project-name game_2048 --auto-inc "make a 2048 game" 1. 使用新的METAGPT_ROOT生成方式，而非寻找git，以便cli安装 2. 命令行由fire换为了typer，它会带来相对更好的体验 3. project_name可以灵活指定了，在metagpt命令行输入中配置 5. 其他 1. 现在支持多国语言了，中文已测试 2. BossRequirement -> UserRequirement 3. 大量错误文本的修正，增加了可读性 4. 中量提示词优化，稍微提升了一些准确率 5. 暂时屏蔽了LongtermMemory相关逻辑，这个逻辑底层调用了langchain的FAISS，会带来~5秒加载耗时 6. 修复了安装包中的部分描述错误 7. 去除了config中在openai_proxy设定时对base的重复修改，这个修改应该在openai初始化时发生 8. 修复了JSON在中文存储时的特定问题，ensure_ascii=False --- examples/debate.py | 2 +- metagpt/actions/action.py | 4 + metagpt/actions/design_api.py | 37 +++--- metagpt/actions/project_management.py | 36 +++--- .../{SummarizeCode.py => summarize_code.py} | 21 ++-- metagpt/actions/write_code.py | 3 +- metagpt/actions/write_code_review.py | 84 +++++++++---- metagpt/actions/write_prd.py | 88 ++++++------- metagpt/actions/write_test.py | 2 +- metagpt/config.py | 6 +- metagpt/document.py | 116 ++++++++++++------ metagpt/environment.py | 30 ++++- metagpt/memory/longterm_memory.py | 2 +- metagpt/provider/openai_api.py | 2 + metagpt/{document_store => }/repo_parser.py | 48 ++++---- metagpt/roles/engineer.py | 26 ++-- metagpt/roles/qa_engineer.py | 4 +- metagpt/roles/role.py | 15 +++ metagpt/startup.py | 7 +- metagpt/team.py | 10 +- tests/metagpt/actions/mock.py | 2 +- tests/metagpt/roles/mock.py | 2 +- tests/metagpt/roles/test_ui.py | 2 +- ...st_software_company.py => test_startup.py} | 13 +- 24 files changed, 359 insertions(+), 203 deletions(-) rename metagpt/actions/{SummarizeCode.py => summarize_code.py} (62%) rename metagpt/{document_store => }/repo_parser.py (67%) rename tests/metagpt/{test_software_company.py => test_startup.py} (51%) diff --git a/examples/debate.py b/examples/debate.py index 0f5d1591b..e62a5aaa1 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -88,7 +88,7 @@ async def debate(idea: str, investment: float = 3.0, n_round: int = 5): team = Team() team.hire([Biden, Trump]) team.invest(investment) - team.start_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first + team.run_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first await team.run(n_round=n_round) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 790295d55..f8016b8a2 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -30,6 +30,10 @@ class Action(ABC): self.desc = "" self.content = "" self.instruct_content = None + self.env = None + + def set_env(self, env): + self.env = env def set_prefix(self, prefix, profile): """Set prefix for later usage""" diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index f58d49495..9e2bfc12c 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -14,7 +14,6 @@ from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown from metagpt.utils.mermaid import mermaid_to_file templates = { @@ -27,11 +26,12 @@ templates = { {format_example} ----- Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirement: Fill in the following missing information based on the context, each section name is a key in json ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores ## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -48,7 +48,7 @@ and only output the json inside this tag, nothing else [CONTENT] { "Implementation approach": "We will ...", - "Python package name": "snake_game", + "project_name": "snake_game", "File list": ["main.py"], "Data structures and interfaces": ' classDiagram @@ -78,12 +78,13 @@ and only output the json inside this tag, nothing else {format_example} ----- Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. +ATTENTION: Output carefully referenced "Format example" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -99,7 +100,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Implementation approach We will ... -## Python package name +## project_name ```python "snake_game" ``` @@ -138,7 +139,7 @@ The requirement is clear to me. OUTPUT_MAPPING = { "Implementation approach": (str, ...), - "Python package name": (str, ...), + "project_name": (str, ...), "File list": (List[str], ...), "Data structures and interfaces": (str, ...), "Program call flow": (str, ...), @@ -170,7 +171,7 @@ class WriteDesign(Action): if context[-1].instruct_content: logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict())) + prd_file.write_text(context[-1].instruct_content.json(ensure_ascii=False), encoding='utf-8') async def _save_system_design(self, docs_path, resources_path, system_design): data_api_design = system_design.instruct_content.dict()[ @@ -183,14 +184,14 @@ class WriteDesign(Action): await mermaid_to_file(seq_flow, resources_path / "seq_flow") system_design_file = docs_path / "system_design.md" logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict()))) + system_design_file.write_text(system_design.instruct_content.json(ensure_ascii=False), encoding='utf-8') async def _save(self, context, system_design): if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["Python package name"] + project_name = system_design.instruct_content.dict()["project_name"] else: - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = CONFIG.workspace_path / ws_name + project_name = CodeParser.parse_str(block="project_name", text=system_design) + workspace = CONFIG.workspace_path / project_name self.recreate_workspace(workspace) docs_path = workspace / "docs" resources_path = workspace / "resources" @@ -204,11 +205,11 @@ class WriteDesign(Action): prompt = prompt_template.format(context=context, format_example=format_example) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python package name" contain space, have to use setattr - setattr( - system_design.instruct_content, - "Python package name", - system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), - ) + # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" contain space, have to use setattr + # setattr( + # system_design.instruct_content, + # "project_name", + # system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), + # ) await self._save(context, system_design) return system_design diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 467cb4d83..805226a25 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -11,7 +11,6 @@ from metagpt.actions.action import Action from metagpt.config import CONFIG from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown templates = { "json": { @@ -23,19 +22,20 @@ templates = { {format_example} ----- Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. +ATTENTION: Output carefully referenced "Format example" in format. ## Required Python third-party packages: Provide Python list[str] in requirements.txt format ## Required Other language third-party packages: Provide Python list[str] in requirements.txt format -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - ## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first ## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. ## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. @@ -52,17 +52,17 @@ and only output the json inside this tag, nothing else "Required Other language third-party packages": [ "No third-party ..." ], + "Logic Analysis": [ + ["game.py", "Contains..."] + ], + "Task list": [ + "game.py" + ], "Full API spec": """ openapi: 3.0.0 ... description: A JSON object ... """, - "Logic Analysis": [ - ["game.py","Contains..."] - ], - "Task list": [ - "game.py" - ], "Shared Knowledge": """ 'game.py' contains ... """, @@ -86,12 +86,12 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Required Other language third-party packages: Provided in requirements.txt format -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - ## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first ## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. ## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. @@ -126,14 +126,16 @@ description: A JSON object ... ## Logic Analysis ```python [ - ["game.py", "Contains ..."], + ["index.js", "Contains ..."], + ["main.py", "Contains ..."], ] ``` ## Task list ```python [ - "game.py", + "index.js", + "main.py", ] ``` @@ -167,11 +169,11 @@ class WriteTasks(Action): def _save(self, context, rsp): if context[-1].instruct_content: - ws_name = context[-1].instruct_content.dict()["Python package name"] + ws_name = context[-1].instruct_content.dict()["project_name"] else: - ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) + ws_name = CodeParser.parse_str(block="project_name", text=context[-1].content) file_path = CONFIG.workspace_path / ws_name / "docs/api_spec_and_tasks.md" - file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) + file_path.write_text(rsp.instruct_content.json(ensure_ascii=False)) # Write requirements.txt requirements_path = CONFIG.workspace_path / ws_name / "requirements.txt" diff --git a/metagpt/actions/SummarizeCode.py b/metagpt/actions/summarize_code.py similarity index 62% rename from metagpt/actions/SummarizeCode.py rename to metagpt/actions/summarize_code.py index 49a350b75..a85d3cdeb 100644 --- a/metagpt/actions/SummarizeCode.py +++ b/metagpt/actions/summarize_code.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ @Author : alexanderwu -@File : SummarizeCode.py +@File : summarize_code.py """ from tenacity import retry, stop_after_attempt, wait_fixed @@ -13,6 +13,7 @@ from metagpt.schema import Message PROMPT_TEMPLATE = """ NOTICE Role: You are a professional software engineer, and your main task is to review the code. +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". ----- @@ -20,13 +21,13 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc {context} ----- -## Code Review All: 请你对历史所有文件进行阅读，分析每个文件是否都完整实现了用户需求，找到可能的bug，如函数未实现、调用错误、未引用等 +## Code Review All: 请你对历史所有文件进行阅读，在文件中找到可能的bug，如函数未实现、调用错误、未引用等 -## Call flow: 根据实现的函数，使用mermaid绘制完整的调用链 +## Call flow: mermaid代码，根据实现的函数，使用mermaid绘制完整的调用链 ## Summary: 根据历史文件的实现情况进行总结 -## TODOs: 这里写出需要修改的文件列表，我们会在之后进行修改 +## TODOs: Python dict[str, str]，这里写出需要修改的文件列表与理由，我们会在之后进行修改 """ @@ -67,15 +68,15 @@ flowchart TB - ... ## TODOs -1. ... -2. ... -3. ... +{ + "a.py": "implement requirement xxx...", +} """ class SummarizeCode(Action): - def __init__(self, name="SummaryCode", context: list[Message] = None, llm=None): + def __init__(self, name="SummarizeCode", context: list[Message] = None, llm=None): super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) @@ -84,8 +85,8 @@ class SummarizeCode(Action): return code_rsp async def run(self, context): - format_example = FORMAT_EXAMPLE.format() + format_example = FORMAT_EXAMPLE prompt = PROMPT_TEMPLATE.format(context=context, format_example=format_example) - logger.info("Code review all..") + logger.info("Summarize code..") rsp = await self.summarize_code(prompt) return rsp diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 1f6d16b3b..2631ec138 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -16,6 +16,7 @@ from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ NOTICE Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". ----- @@ -60,7 +61,7 @@ class WriteCode(Action): design = [i for i in context if i.cause_by == WriteDesign][0] - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) + ws_name = CodeParser.parse_str(block="project_name", text=design.content) ws_path = CONFIG.workspace_path / ws_name if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): ws_path = ws_path / ws_name diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index c6538bf7b..aebe3f4fa 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -6,56 +6,84 @@ @File : write_code_review.py """ +from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed +from metagpt.config import CONFIG PROMPT_TEMPLATE = """ NOTICE Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code Review: Based on the following context and code, follow the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. -1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. -2. Are there any issues with the code logic? If so, how to solve it? -3. Does the existing code follow the "Data structures and interfaces"? -4. Is there a function in the code that is not fully implemented? If so, how to implement it? -5. Does the code have unnecessary or lack dependencies? If so, how to solve it? - -## Rewrite Code: rewrite {filename} based on "Code Review" with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. ------ # Context {context} -## Code: {filename} +## Code to be Reviewed: {filename} ``` {code} ``` + ----- +## Code Review: Based on the "Code to be Reviewed", provide key, clear, concise, and specific code modification suggestions, up to 5. +1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. +2. Is the code logic completely correct? If there are errors, please indicate how to correct them. +3. Does the existing code follow the "Data structures and interfaces"? +4. Are all functions implemented? If there is no implementation, please indicate how to achieve it step by step. +5. Have all necessary pre-dependencies been imported? If not, indicate which ones need to be imported +6. Is the code implemented concisely enough? Are methods from other files being reused correctly? + +## Code Review Result: If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM. +LGTM/LBTM + +## Rewrite Code: if it still has some bugs, rewrite {filename} based on "Code Review" with triple quotes, try to get LGTM. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. RETURN ALL CODE, NEVER OMIT ANYTHING. 以任何方式省略代码都是不允许的。 +``` +``` + ## Format example ------ {format_example} ------ """ FORMAT_EXAMPLE = """ - -## Code Review +----- +# EXAMPLE 1 +## Code Review: {filename} 1. No, we should add the logic of ... 2. ... 3. ... 4. ... 5. ... +6. ... + +## Code Review Result: {filename} +LBTM ## Rewrite Code: {filename} ```python ## {filename} ... ``` +----- +# EXAMPLE 2 +## Code Review: {filename} +1. Yes. +2. Yes. +3. Yes. +4. Yes. +5. Yes. +6. Yes. + +## Code Review Result: {filename} +LGTM + +## Rewrite Code: {filename} +pass +----- """ @@ -64,17 +92,27 @@ class WriteCodeReview(Action): super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def write_code(self, prompt): + async def write_code_review_and_rewrite(self, prompt): code_rsp = await self._aask(prompt) + result = CodeParser.parse_block("Code Review Result", code_rsp) + if "LGTM" in result: + return result, None code = CodeParser.parse_code(block="", text=code_rsp) - return code + return result, code async def run(self, context, code, filename): - format_example = FORMAT_EXAMPLE.format(filename=filename) - prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example) - logger.info(f'Code review {filename}..') - code = await self.write_code(prompt) + iterative_code = code + k = CONFIG.code_review_k_times + for i in range(k): + format_example = FORMAT_EXAMPLE.format(filename=filename) + prompt = PROMPT_TEMPLATE.format(context=context, code=iterative_code, filename=filename, format_example=format_example) + logger.info(f'Code review and rewrite {filename}: {i+1}/{k} | {len(iterative_code)=}, {len(code)=}') + result, rewrited_code = await self.write_code_review_and_rewrite(prompt) + if "LBTM" in result: + iterative_code = rewrited_code + elif "LGTM" in result: + return iterative_code # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) - return code - \ No newline at end of file + # 如果rewrited_code是None（原code perfect），那么直接返回code + return iterative_code diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 584d31998..4780762ca 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -17,54 +17,50 @@ templates = { "json": { "PROMPT_TEMPLATE": """ # Context -## Original Requirements -{requirements} - -## Search Information -{search_information} - -## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME -```mermaid -quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] - "Campaign B": [0.45, 0.23] - "Campaign C": [0.57, 0.69] - "Campaign D": [0.78, 0.34] - "Campaign E": [0.40, 0.34] - "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6] -``` +{{ + "Original Requirements": "{requirements}", + "Search Information": "" +}} ## Format example {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, each section name is a key in json +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. +Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. +ATTENTION: Output carefully referenced "Format example" in format. -## Original Requirements: Provide as Plain text, place the polished complete original requirements here +## YOU NEED TO FULFILL THE BELOW JSON DOC -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories - -## Competitive Analysis: Provided as Python list[str], up to 8 competitive product analyses - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards - -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. - -## Anything UNCLEAR: Provide as Plain text. Try to clarify it. +{{ + "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. + "Original Requirements": "", # str, place the polished complete original requirements here + "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Search Information": "", + "Requirements": "", + "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. + "User Stories": [], # Provided as Python list[str], up to 5 scenario-based user stories + "Competitive Analysis": [], # Provided as Python list[str], up to 8 competitive product analyses + # Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + "Competitive Quadrant Chart": "quadrantChart + title Reach and engagement of campaigns + x-axis Low Reach --> High Reach + y-axis Low Engagement --> High Engagement + quadrant-1 We should expand + quadrant-2 Need to promote + quadrant-3 Re-evaluate + quadrant-4 May be improved + Campaign A: [0.3, 0.6] + Campaign B: [0.45, 0.23] + Campaign C: [0.57, 0.69] + Campaign D: [0.78, 0.34] + Campaign E: [0.40, 0.34] + Campaign F: [0.35, 0.78]", + "Requirement Analysis": "", # Provide as Plain text. + "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], # Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards + "UI Design draft": "", # Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. + "Anything UNCLEAR": "", # Provide as Plain text. Try to clarify it. +}} output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -72,6 +68,7 @@ and only output the json inside this tag, nothing else "FORMAT_EXAMPLE": """ [CONTENT] { + "Language": "", "Original Requirements": "", "Search Information": "", "Requirements": "", @@ -132,9 +129,12 @@ quadrantChart {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product +Language: Please use the same language as the user requirement to answer, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. +## Language: Provide as Plain text, use the same language as the user requirement. + ## Original Requirements: Provide as Plain text, place the polished complete original requirements here ## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -207,6 +207,7 @@ There are no unclear points. } OUTPUT_MAPPING = { + "Language": (str, ...), "Original Requirements": (str, ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), @@ -232,11 +233,14 @@ class WritePRD(Action): logger.info(sas.result) logger.info(rsp) + # logger.info(format) prompt_template, format_example = get_template(templates, format) + # logger.info(prompt_template) + # logger.info(format_example) prompt = prompt_template.format( requirements=requirements, search_information=info, format_example=format_example ) - logger.debug(prompt) + # logger.info(prompt) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) return prd diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 2f4988c09..9988fda16 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -3,7 +3,7 @@ """ @Time : 2023/5/11 22:12 @Author : alexanderwu -@File : environment.py +@File : write_test.py """ from metagpt.actions.action import Action from metagpt.logs import logger diff --git a/metagpt/config.py b/metagpt/config.py index 1a9cdb4d2..d30a337e3 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -54,10 +54,7 @@ class Config(metaclass=Singleton): (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") - openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy - if openai_proxy: - openai.proxy = openai_proxy - openai.api_base = self.openai_api_base + self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) @@ -87,6 +84,7 @@ class Config(metaclass=Singleton): logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 + self.code_review_k_times = 2 self.puppeteer_config = self._get("PUPPETEER_CONFIG", "") self.mmdc = self._get("MMDC", "mmdc") diff --git a/metagpt/document.py b/metagpt/document.py index 044210218..cf0821421 100644 --- a/metagpt/document.py +++ b/metagpt/document.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : document.py """ - +from enum import Enum from typing import Union, Optional from pathlib import Path from pydantic import BaseModel, Field @@ -18,7 +18,9 @@ from langchain.document_loaders import ( from langchain.text_splitter import CharacterTextSplitter from tqdm import tqdm +from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.repo_parser import RepoParser def validate_cols(content_col: str, df: pd.DataFrame): @@ -48,42 +50,56 @@ def read_data(data_path: Path): return data +class DocumentStatus(Enum): + """Indicates document status, a mechanism similar to RFC/PEP""" + DRAFT = "draft" + UNDERREVIEW = "underreview" + APPROVED = "approved" + DONE = "done" + + class Document(BaseModel): """ Document: Handles operations related to document files. """ - content: str = Field(default='') - file_path: Path = Field(default=None) + path: Path = Field(default=None) + name: str = Field(default="") + content: str = Field(default="") + + # metadata? in content perhaps. + author: str = Field(default="") + status: DocumentStatus = Field(default=DocumentStatus.DRAFT) + reviews: list = Field(default_factory=list) @classmethod - def from_path(cls, file_path: Path): + def from_path(cls, path: Path): """ Create a Document instance from a file path. """ - if not file_path.exists(): - raise FileNotFoundError(f"File {file_path} not found.") - content = file_path.read_text() - return cls(content=content, file_path=file_path) + if not path.exists(): + raise FileNotFoundError(f"File {path} not found.") + content = path.read_text() + return cls(content=content, path=path) @classmethod - def from_text(cls, text: str, file_path: Optional[Path] = None): + def from_text(cls, text: str, path: Optional[Path] = None): """ Create a Document from a text string. """ - return cls(content=text, file_path=file_path) + return cls(content=text, path=path) - def to_path(self, file_path: Optional[Path] = None): + def to_path(self, path: Optional[Path] = None): """ Save content to the specified file path. """ - if file_path is not None: - self.file_path = file_path + if path is not None: + self.path = path - if self.file_path is None: + if self.path is None: raise ValueError("File path is not set.") - self.file_path.parent.mkdir(parents=True, exist_ok=True) - self.file_path.write_text(self.content) + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(self.content, encoding="utf-8") def persist(self): """ @@ -140,25 +156,35 @@ class IndexableDocument(Document): raise NotImplementedError("Data type not supported for metadata extraction.") +class RepoMetadata(BaseModel): + + name: str = Field(default="") + n_docs: int = Field(default=0) + n_chars: int = Field(default=0) + symbols: list = Field(default_factory=list) + + class Repo(BaseModel): # Name of this repo. name: str = Field(default="") + # metadata: RepoMetadata = Field(default=RepoMetadata) docs: dict[Path, Document] = Field(default_factory=dict) codes: dict[Path, Document] = Field(default_factory=dict) assets: dict[Path, Document] = Field(default_factory=dict) - repo_path: Path = Field(default_factory=Path) + path: Path = Field(default=None) def _path(self, filename): - return self.repo_path / filename + return self.path / filename @classmethod - def from_path(cls, repo_path: Path): + def from_path(cls, path: Path): """Load documents, code, and assets from a repository path.""" - repo_path.mkdir(parents=True, exist_ok=True) - repo = Repo(repo_path = repo_path) - for file_path in repo_path.rglob('*'): - if file_path.is_file(): + path.mkdir(parents=True, exist_ok=True) + repo = Repo(path=path, name=path.name) + for file_path in path.rglob('*'): + # FIXME: These judgments are difficult to support multiple programming languages and need to be more general + if file_path.is_file() and file_path.suffix in [".json", ".txt", ".md", ".py", ".js", ".css", ".html"]: repo._set(file_path.read_text(), file_path) return repo @@ -171,23 +197,24 @@ class Repo(BaseModel): for asset in self.assets.values(): asset.to_path() - def _set(self, content: str, file_path: Path): + def _set(self, content: str, path: Path): """Add a document to the appropriate category based on its file extension.""" - file_ext = file_path.suffix + suffix = path.suffix + doc = Document(content=content, path=path, name=str(path.relative_to(self.path))) - doc = Document(content=content, file_path=file_path) - if file_ext.lower() == '.md': - self.docs[file_path] = doc - elif file_ext.lower() in ['.py', '.js', '.css', '.html']: - self.codes[file_path] = doc + # FIXME: These judgments are difficult to support multiple programming languages and need to be more general + if suffix.lower() == '.md': + self.docs[path] = doc + elif suffix.lower() in ['.py', '.js', '.css', '.html']: + self.codes[path] = doc else: - self.assets[file_path] = doc + self.assets[path] = doc return doc def set(self, content: str, filename: str): """Set a document and persist it to disk.""" - file_path = self._path(filename) - doc = self._set(content, file_path) + path = self._path(filename) + doc = self._set(content, path) doc.to_path() def get(self, filename: str) -> Optional[Document]: @@ -195,13 +222,32 @@ class Repo(BaseModel): path = self._path(filename) return self.docs.get(path) or self.codes.get(path) or self.assets.get(path) + def get_text_documents(self) -> list[Document]: + return list(self.docs.values()) + list(self.codes.values()) -def main(): - repo1 = Repo.from_path(Path("/Users/alexanderwu/workspace/t1")) + def eda(self) -> RepoMetadata: + n_docs = sum(len(i) for i in [self.docs, self.codes, self.assets]) + n_chars = sum(sum(len(j.content) for j in i.values()) for i in [self.docs, self.codes, self.assets]) + symbols = RepoParser(base_directory=self.path).generate_symbols() + return RepoMetadata(name=self.name, n_docs=n_docs, n_chars=n_chars, symbols=symbols) + + +def set_existing_repo(path=CONFIG.workspace_path / "t1"): + repo1 = Repo.from_path(path) repo1.set("wtf content", "doc/wtf_file.md") repo1.set("wtf code", "code/wtf_file.py") logger.info(repo1) # check doc +def load_existing_repo(path=CONFIG.workspace_path / "web_tetris"): + repo = Repo.from_path(path) + logger.info(repo) + logger.info(repo.eda()) + + +def main(): + load_existing_repo() + + if __name__ == '__main__': main() diff --git a/metagpt/environment.py b/metagpt/environment.py index 38077c90d..44c9b1c67 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -7,10 +7,12 @@ """ import asyncio from typing import Iterable +from pathlib import Path from pydantic import BaseModel, Field # from metagpt.document import Document +from metagpt.logs import logger from metagpt.document import Repo from metagpt.memory import Memory from metagpt.roles import Role @@ -26,6 +28,7 @@ class Environment(BaseModel): memory: Memory = Field(default_factory=Memory) history: str = Field(default='') repo: Repo = Field(default_factory=Repo) + kv: dict = Field(default_factory=dict) class Config: arbitrary_types_allowed = True @@ -52,9 +55,32 @@ class Environment(BaseModel): self.memory.add(message) self.history += f"\n{message}" - def publish_doc(self, content: str, filename: str): + def set_doc(self, content: str, filename: str): """向当前环境发布文档（包括代码）""" - self.repo.set(content, filename) + return self.repo.set(content, filename) + + def get_doc(self, filename: str): + return self.repo.get(filename) + + def set(self, k: str, v: str): + self.kv[k] = v + + def get(self, k: str): + return self.kv.get(k, None) + + def load_existing_repo(self, path: Path, inc: bool): + self.repo = Repo.from_path(path) + logger.info(self.repo.eda()) + + # Incremental mode: publish all docs to messages. Then roles can read the docs. + if inc: + docs = self.repo.get_text_documents() + for doc in docs: + msg = Message(content=doc.content) + self.publish_message(msg) + logger.info(f"Message from existing doc {doc.path}: {msg}") + logger.info(f"Load {len(docs)} docs from existing repo.") + raise NotImplementedError async def run(self, k=1): """处理一次所有信息的运行 diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index f8abea5f3..b21f80b7d 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -28,7 +28,7 @@ class LongTermMemory(Memory): logger.warning(f"It may the first time to run Agent {role_id}, the long-term memory is empty") else: logger.warning( - f"Agent {role_id} has existed memory storage with {len(messages)} messages " f"and has recovered them." + f"Agent {role_id} has existing memory storage with {len(messages)} messages " f"and has recovered them." ) self.msg_from_recover = True self.add_batch(messages) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 34e5693f8..8ac0c4b21 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -157,6 +157,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if config.openai_api_type: openai.api_type = config.openai_api_type openai.api_version = config.openai_api_version + if config.openai_proxy: + openai.proxy = config.openai_proxy self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: diff --git a/metagpt/document_store/repo_parser.py b/metagpt/repo_parser.py similarity index 67% rename from metagpt/document_store/repo_parser.py rename to metagpt/repo_parser.py index f7e2b0f4a..0020d47aa 100644 --- a/metagpt/document_store/repo_parser.py +++ b/metagpt/repo_parser.py @@ -6,15 +6,19 @@ @File : repo_parser.py """ import json -import pathlib +from pathlib import Path + import ast - import pandas as pd +from pydantic import BaseModel, Field +from pprint import pformat + +from metagpt.config import CONFIG +from metagpt.logs import logger -class RepoParser: - def __init__(self): - self.base_directory = None +class RepoParser(BaseModel): + base_directory: Path = Field(default=None) def parse_file(self, file_path): """Parse a Python file in the repository.""" @@ -38,43 +42,42 @@ class RepoParser: file_info["classes"].append({"name": node.name, "methods": class_methods}) elif is_func(node): file_info["functions"].append(node.name) - elif isinstance(node, ast.Assign) or isinstance(node, ast.AnnAssign): + elif isinstance(node, (ast.Assign, ast.AnnAssign)): for target in node.targets if isinstance(node, ast.Assign) else [node.target]: if isinstance(target, ast.Name): file_info["globals"].append(target.id) return file_info - def generate_json_structure(self, directory, output_path): - """Generate a JSON file documenting the repository structure.""" + def generate_symbols(self): files_classes = [] + directory = self.base_directory for path in directory.rglob('*.py'): tree = self.parse_file(path) file_info = self.extract_class_and_function_info(tree, path) files_classes.append(file_info) + return files_classes + + def generate_json_structure(self, output_path): + """Generate a JSON file documenting the repository structure.""" + files_classes = self.generate_symbols() output_path.write_text(json.dumps(files_classes, indent=4)) - def generate_dataframe_structure(self, directory, output_path): + def generate_dataframe_structure(self, output_path): """Generate a DataFrame documenting the repository structure and save as CSV.""" - files_classes = [] - for path in directory.rglob('*.py'): - tree = self.parse_file(path) - file_info = self.extract_class_and_function_info(tree, path) - files_classes.append(file_info) - + files_classes = self.generate_symbols() df = pd.DataFrame(files_classes) df.to_csv(output_path, index=False) - def generate_structure(self, directory_path, output_path=None, mode='json'): + def generate_structure(self, output_path=None, mode='json'): """Generate the structure of the repository as a specified format.""" - self.base_directory = pathlib.Path(directory_path) output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}" - output_path = pathlib.Path(output_path) if output_path else output_file + output_path = Path(output_path) if output_path else output_file if mode == 'json': - self.generate_json_structure(self.base_directory, output_path) + self.generate_json_structure(output_path) elif mode == 'csv': - self.generate_dataframe_structure(self.base_directory, output_path) + self.generate_dataframe_structure(output_path) def is_func(node): @@ -82,8 +85,9 @@ def is_func(node): def main(): - repo_parser = RepoParser() - repo_parser.generate_structure("/Users/alexanderwu/git/mg1/metagpt", "/Users/alexanderwu/git/mg1/mg1-structure.csv", mode='csv') + repo_parser = RepoParser(base_directory=CONFIG.workspace_path / "web_2048") + symbols = repo_parser.generate_symbols() + logger.info(pformat(symbols)) if __name__ == '__main__': diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 171af47f0..e3f36b50d 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -11,7 +11,7 @@ from collections import OrderedDict from pathlib import Path from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.actions.SummarizeCode import SummarizeCode +from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role @@ -74,8 +74,8 @@ class Engineer(Role): super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review - if self.use_code_review: - self._init_actions([WriteCode, WriteCodeReview]) + # if self.use_code_review: + # self._init_actions([WriteCode, WriteCodeReview]) self._watch([WriteTasks]) self.todos = [] self.n_borg = n_borg @@ -93,8 +93,8 @@ class Engineer(Role): @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) + return system_design_msg.instruct_content.dict().get("project_name").strip().strip("'").strip('"') + return CodeParser.parse_str(block="project_name", text=system_design_msg.content) def get_workspace(self) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] @@ -182,16 +182,16 @@ class Engineer(Role): msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) for m in msg: context.append(m.content) - context_str = "\n".join(context) + context_str = "\n----------\n".join(context) # Write code code = await WriteCode().run(context=context_str, filename=todo) # Code review if self.use_code_review: - try: - rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) + # try: + rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) + code = rewrite_code + # except Exception as e: + # logger.error("code review failed!", e) file_path = self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) @@ -203,8 +203,8 @@ class Engineer(Role): msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) for m in msg: context.append(m.content) - context_str = "\n".join(context) - code_review_all = await SummarizeCode().run(context=context_str) + context_str = "\n----------\n".join(context) + summary = await SummarizeCode().run(context=context_str) logger.info(f"Done {self.get_workspace()} generating.") msg = Message( diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index f124646b3..313fe4aba 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -45,8 +45,8 @@ class QaEngineer(Role): @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name") - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) + return system_design_msg.instruct_content.dict().get("project_name") + return CodeParser.parse_str(block="project_name", text=system_design_msg.content) def get_workspace(self, return_proj_dir=True) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index d772c0748..5c5e7b76d 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -50,6 +50,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi {name}: {result} """ + class RoleReactMode(str, Enum): REACT = "react" BY_ORDER = "by_order" @@ -59,6 +60,7 @@ class RoleReactMode(str, Enum): def values(cls): return [item.value for item in cls] + class RoleSetting(BaseModel): """Role Settings""" name: str @@ -131,6 +133,7 @@ class Role: logger.warning(f"is_human attribute does not take effect," f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") i = action + i.set_env(self._rc.env) i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) self._states.append(f"{idx}. {action}") @@ -172,6 +175,18 @@ class Role: """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" self._rc.env = env + def set_doc(self, content: str, filename: str): + return self._rc.env.set_doc(content, filename) + + def get_doc(self, filename: str): + return self._rc.env.get_doc(filename) + + def set(self, k, v): + return self._rc.env.set(k, v) + + def get(self, k): + return self._rc.env.get(k) + @property def profile(self): """Get the role description (position)""" diff --git a/metagpt/startup.py b/metagpt/startup.py index d8ca4072f..38f457fc2 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -15,7 +15,8 @@ def startup( code_review: bool = typer.Option(True, help="Whether to use code review."), run_tests: bool = typer.Option(False, help="Whether to enable QA for adding & running tests."), implement: bool = typer.Option(True, help="Enable or disable code implementation."), - project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'"), + project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'."), + inc: bool = typer.Option(False, help="Incremental mode. Use it to coop with existing repo."), ): """Run a startup. Be a boss.""" from metagpt.roles import ProductManager, Architect, ProjectManager, Engineer, QaEngineer @@ -37,9 +38,9 @@ def startup( company.hire([QaEngineer()]) company.invest(investment) - company.start_project(project_name, idea) + company.run_project(idea, project_name=project_name, inc=inc) asyncio.run(company.run(n_round=n_round)) if __name__ == "__main__": - app() + startup(idea="Make a 2048 game.") diff --git a/metagpt/team.py b/metagpt/team.py index 2332aaa46..a22a09fe4 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -42,15 +42,19 @@ class Team(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, project_name, idea, send_to: str = ""): + def run_project(self, idea, send_to: str = "", project_name: str = "", inc: bool = False): """Start a project from publishing user requirement.""" self.idea = idea # If user set project_name, then use it. - self.env.repo.name = project_name + if project_name: + path = CONFIG.workspace_path / project_name + self.env.load_existing_repo(path, inc=inc) + + # Human requirement. self.env.publish_message(Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to)) def _save(self): - logger.info(self.json()) + logger.info(self.json(ensure_ascii=False)) async def run(self, n_round=3): """Run company until target round or no money""" diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index 5be1d8001..d367e253e 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -90,7 +90,7 @@ Python's in-built data structures like lists and dictionaries will be used exten For testing, we can use the PyTest framework. This is a mature full-featured Python testing tool that helps you write better programs. -## Python package name: +## project_name: ```python "adventure_game" ``` diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index fbad06acb..c06844389 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -71,7 +71,7 @@ PRD = '''## 原始需求 ``` ''' -SYSTEM_DESIGN = '''## Python package name +SYSTEM_DESIGN = '''## project_name ```python "smart_search_engine" ``` diff --git a/tests/metagpt/roles/test_ui.py b/tests/metagpt/roles/test_ui.py index d58d31bd9..ec507f75d 100644 --- a/tests/metagpt/roles/test_ui.py +++ b/tests/metagpt/roles/test_ui.py @@ -18,5 +18,5 @@ async def test_ui_role(idea: str, investment: float = 3.0, n_round: int = 5): company = Team() company.hire([ProductManager(), UI()]) company.invest(investment) - company.start_project(idea) + company.run_project(idea) await company.run(n_round=n_round) diff --git a/tests/metagpt/test_software_company.py b/tests/metagpt/test_startup.py similarity index 51% rename from tests/metagpt/test_software_company.py rename to tests/metagpt/test_startup.py index 4fc651f52..53a8d8735 100644 --- a/tests/metagpt/test_software_company.py +++ b/tests/metagpt/test_startup.py @@ -3,17 +3,26 @@ """ @Time : 2023/5/15 11:40 @Author : alexanderwu -@File : test_software_company.py +@File : test_startup.py """ import pytest +from typer.testing import CliRunner + +runner = CliRunner() from metagpt.logs import logger from metagpt.team import Team +from metagpt.startup import app @pytest.mark.asyncio async def test_team(): company = Team() - company.start_project("做一个基础搜索引擎，可以支持知识库") + company.run_project("做一个基础搜索引擎，可以支持知识库") history = await company.run(n_round=5) logger.info(history) + + +# def test_startup(): +# args = ["Make a 2048 game"] +# result = runner.invoke(app, args) From 70a0be3300a3b7e3d40a7ae55692f36ab8e46ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:49:05 +0800 Subject: [PATCH 098/135] feat: +annotation --- metagpt/const.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/const.py b/metagpt/const.py index 49965b622..a8c7356ca 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -4,8 +4,9 @@ @Time : 2023/5/1 11:59 @Author : alexanderwu @File : const.py -@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, added key definitions for +@Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for common properties in the Message. +@Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. """ from pathlib import Path From ad6cf62d2113bb16b9ce93656d353dcbe06a2657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 15:57:19 +0800 Subject: [PATCH 099/135] feat: +annotation --- metagpt/roles/engineer.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index b6ecc4767..3cf1f2125 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -10,6 +10,9 @@ 2. Consolidate message reception and processing logic within `_observe`. 3. Fix bug: Add logic for handling asynchronous message processing when messages are not ready. 4. Supplemented the external transmission of internal messages. +@Modified By: mashenquan, 2023-11-27. + 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. + 2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality. """ from __future__ import annotations @@ -97,11 +100,11 @@ class Engineer(Role): async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" changed_files = await self._act_sp_precision(review=self.use_code_review) - # 仅单测 + # Unit tests only. if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: changed_files.add(CONFIG.REQA_FILENAME) - from metagpt.roles import QaEngineer # 避免循环引用 + from metagpt.roles import QaEngineer # Avoid circular references. msg = Message( content="\n".join(changed_files), @@ -122,7 +125,7 @@ class Engineer(Role): design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_files = Documents() - # 由上游变化导致的recode + # Recode caused by upstream changes. for filename in changed_task_files: design_doc = await design_file_repo.get(filename) task_doc = await task_file_repo.get(filename) @@ -144,7 +147,7 @@ class Engineer(Role): ) changed_files.docs[task_filename] = coding_doc self.todos = [WriteCode(context=i, llm=self._llm) for i in changed_files.docs.values()] - # 用户直接修改的code + # Code directly modified by the user. dependency = await CONFIG.git_repo.get_dependency() for filename in changed_src_files: if filename in changed_files.docs: From 94ab03d2daf66bc81db70a8a7f30323555f86ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:02:18 +0800 Subject: [PATCH 100/135] feat: +annotation --- metagpt/roles/product_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 81577ec2c..bc6771829 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : product_manager.py +@Modified By: mashenquan, 2023/11/27. Add `PrepareDocuments` action according to Section 2.2.3.5.1 of RFC 135. """ from metagpt.actions import BossRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments From 41549e628082156e1bd613aa337665e95c2d685c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:11:49 +0800 Subject: [PATCH 101/135] feat: +annotation --- metagpt/roles/qa_engineer.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index a88b01e37..763ab6a3f 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -6,6 +6,11 @@ @File : qa_engineer.py @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, modify the data type of the `cause_by` value in the `Message` to a string, and utilize the new message filtering feature. +@Modified By: mashenquan, 2023-11-27. + 1. Following the think-act principle, solidify the task parameters when creating the + WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function. + 2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message + to using file references. """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest from metagpt.config import CONFIG From e656e55f304f79a05591bad33dd88df6230b0d53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:15:55 +0800 Subject: [PATCH 102/135] feat: +annotation --- metagpt/schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index e910fc866..959e70dc1 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -6,7 +6,10 @@ @File : schema.py @Modified By: mashenquan, 2023-10-31. According to Chapter 2.2.1 of RFC 116: Replanned the distribution of responsibilities and functional positioning of `Message` class attributes. -@Modified By: mashenquan, 2023/11/22. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. +@Modified By: mashenquan, 2023/11/22. + 1. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. + 2. Encapsulate the common key-values set to pydantic structures to standardize and unify parameter passing + between actions. """ from __future__ import annotations From 4c296a348b0f59b5889f9baf207ff0b91619b982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:18:39 +0800 Subject: [PATCH 103/135] feat: +annotation --- metagpt/software_company.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 5aa0864e0..72f28ab1d 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,6 +4,8 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py +@Modified By: mashenquan, 2023/11/27. Add an archiving operation after completing the project, as specified in + Section 2.2.3.3 of RFC 135. """ from pydantic import BaseModel, Field From 0c84c2c212bf6892ecbd9d4fb28c2135a881379c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:21:53 +0800 Subject: [PATCH 104/135] feat: +annotation --- metagpt/utils/common.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 9002a8dfb..fd3958a61 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -6,6 +6,8 @@ @File : common.py @Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116: Add generic class-to-string and object-to-string conversion functionality. +@Modified By: mashenquan, 2023/11/27. Bug fix: `parse_recipient` failed to parse the recipient in certain GPT-3.5 + responses. """ import ast import contextlib From bd5daeb4e6743dd859d1d341f39eb95efcfa1b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 16:27:35 +0800 Subject: [PATCH 105/135] feat: +annotation --- metagpt/utils/dependency_file.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index 429027c7a..653e07ef9 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -18,11 +18,21 @@ from metagpt.logs import logger class DependencyFile: + """A class representing a DependencyFile for managing dependencies. + + :param workdir: The working directory path for the DependencyFile. + """ + def __init__(self, workdir: Path | str): + """Initialize a DependencyFile instance. + + :param workdir: The working directory path for the DependencyFile. + """ self._dependencies = {} self._filename = Path(workdir) / ".dependencies.json" async def load(self): + """Load dependencies from the file asynchronously.""" if not self._filename.exists(): return try: @@ -33,6 +43,7 @@ class DependencyFile: logger.error(f"Failed to load {str(self._filename)}, error:{e}") async def save(self): + """Save dependencies to the file asynchronously.""" try: data = json.dumps(self._dependencies) async with aiofiles.open(str(self._filename), mode="w") as writer: @@ -41,6 +52,12 @@ class DependencyFile: logger.error(f"Failed to save {str(self._filename)}, error:{e}") async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): + """Update dependencies for a file asynchronously. + + :param filename: The filename or path. + :param dependencies: The set of dependencies. + :param persist: Whether to persist the changes immediately. + """ if persist: await self.load() @@ -65,6 +82,12 @@ class DependencyFile: await self.save() async def get(self, filename: Path | str, persist=False): + """Get dependencies for a file asynchronously. + + :param filename: The filename or path. + :param persist: Whether to load dependencies from the file immediately. + :return: A set of dependencies. + """ if persist: await self.load() @@ -76,8 +99,10 @@ class DependencyFile: return set(self._dependencies.get(str(key), {})) def delete_file(self): + """Delete the dependency file.""" self._filename.unlink(missing_ok=True) @property def exists(self): + """Check if the dependency file exists.""" return self._filename.exists() From 66fc1b83509cf0d85556a339c21941232ad2934c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 17:43:20 +0800 Subject: [PATCH 106/135] feat: merge geekan:main --- ...mit-config.yaml => .pre-commit-config.yam_ | 0 README.md | 311 ++++-------------- config/config.yaml | 5 +- docs/README_CN.md | 215 ++++-------- docs/README_JA.md | 33 +- examples/build_customized_agent.py | 60 +--- examples/debate.py | 99 ++---- metagpt/config.py | 9 +- metagpt/const.py | 9 +- metagpt/llm.py | 29 +- metagpt/provider/base_chatbot.py | 1 + metagpt/provider/base_gpt_api.py | 55 +++- metagpt/provider/openai_api.py | 84 ++++- metagpt/roles/engineer.py | 1 + metagpt/roles/invoice_ocr_assistant.py | 26 +- metagpt/roles/researcher.py | 19 +- metagpt/roles/role.py | 126 +++++-- metagpt/software_company.py | 59 +--- metagpt/utils/mermaid.py | 5 +- metagpt/utils/token_counter.py | 6 +- requirements.txt | 4 +- setup.py | 6 +- startup.py | 4 +- tests/metagpt/roles/test_ui.py | 4 +- tests/metagpt/test_software_company.py | 6 +- 25 files changed, 488 insertions(+), 688 deletions(-) rename .pre-commit-config.yaml => .pre-commit-config.yam_ (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yam_ similarity index 100% rename from .pre-commit-config.yaml rename to .pre-commit-config.yam_ diff --git a/README.md b/README.md index 70460ceb4..e80082a3a 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,13 @@ # MetaGPT: The Multi-Agent Framework CN doc EN doc JA doc -Discord Follow +Discord Follow License: MIT roadmap Twitter Follow

- AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces Hugging Face @@ -33,132 +32,38 @@ # MetaGPT: The Multi-Agent Framework

Software Company Multi-Role Schematic (Gradually Implementing)

-## MetaGPT's Abilities -https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 +## Install - - -## Examples (fully generated by GPT-4) - -For example, if you type `python startup.py "Design a RecSys like Toutiao"`, you would get many outputs, one of them is data & api design - -![Jinri Toutiao Recsys Data & API Design](docs/resources/workspace/content_rec_sys/resources/data_api_design.png) - -It costs approximately **$0.2** (in GPT-4 API fees) to generate one example with analysis and design, and around **$2.0** for a full project. - - - - -## Installation - -### Installation Video Guide - -- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) - -### Traditional Installation +### Pip installation ```bash -# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) +# Step 1: Ensure that Python 3.9+ is installed on your system. You can check this by using: +# You can use conda to initialize a new python env +# conda create -n metagpt python=3.9 +# conda activate metagpt +python3 --version + +# Step 2: Clone the repository to your local machine for latest version, and install it. +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT +pip3 install -e. # or pip3 install metagpt # for stable version + +# Step 3: run the startup.py +# setup your OPENAI_API_KEY in key.yaml copy from config.yaml +python3 startup.py "Write a cli snake game" + +# Step 4 [Optional]: If you want to save the artifacts like diagrams such as quadrant chart, system designs, sequence flow in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step. +# If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) npm --version sudo npm install -g @mermaid-js/mermaid-cli - -# Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using: -python --version - -# Step 3: Clone the repository to your local machine, and install it. -git clone https://github.com/geekan/metagpt -cd metagpt -pip install -e. ``` -**Note:** +detail installation please refer to [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) -- If already have Chrome, Chromium, or MS Edge installed, you can skip downloading Chromium by setting the environment variable - `PUPPETEER_SKIP_CHROMIUM_DOWNLOAD` to `true`. - -- Some people are [having issues](https://github.com/mermaidjs/mermaid.cli/issues/15) installing this tool globally. Installing it locally is an alternative solution, - - ```bash - npm install @mermaid-js/mermaid-cli - ``` - -- don't forget to the configuration for mmdc in config.yml - - ```yml - PUPPETEER_CONFIG: "./config/puppeteer-config.json" - MMDC: "./node_modules/.bin/mmdc" - ``` - -- if `pip install -e.` fails with error `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`, try instead running `pip install -e. --user` - -- To convert Mermaid charts to SVG, PNG, and PDF formats. In addition to the Node.js version of Mermaid-CLI, you now have the option to use Python version Playwright, pyppeteer or mermaid.ink for this task. - - - Playwright - - **Install Playwright** - - ```bash - pip install playwright - ``` - - - **Install the Required Browsers** - - to support PDF conversion, please install Chrominum. - - ```bash - playwright install --with-deps chromium - ``` - - - **modify `config.yaml`** - - uncomment MERMAID_ENGINE from config.yaml and change it to `playwright` - - ```yaml - MERMAID_ENGINE: playwright - ``` - - - pyppeteer - - **Install pyppeteer** - - ```bash - pip install pyppeteer - ``` - - - **Use your own Browsers** - - pyppeteer allows you use installed browsers, please set the following envirment - - ```bash - export PUPPETEER_EXECUTABLE_PATH = /path/to/your/chromium or edge or chrome - ``` - - please do not use this command to install browser, it is too old - - ```bash - pyppeteer-install - ``` - - - **modify `config.yaml`** - - uncomment MERMAID_ENGINE from config.yaml and change it to `pyppeteer` - - ```yaml - MERMAID_ENGINE: pyppeteer - ``` - - - mermaid.ink - - **modify `config.yaml`** - - uncomment MERMAID_ENGINE from config.yaml and change it to `ink` - - ```yaml - MERMAID_ENGINE: ink - ``` - - Note: this method does not support pdf export. - -### Installation by Docker +### Docker installation +> Note: In the Windows, you need to replace "/opt/metagpt" with a directory that Docker has permission to create, such as "D:\Users\x\metagpt" ```bash # Step 1: Download metagpt official image and prepare config.yaml @@ -174,141 +79,41 @@ # Step 2: Run metagpt demo with container -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ python startup.py "Write a cli snake game" - -# You can also start a container and execute commands in it -docker run --name metagpt -d \ - --privileged \ - -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ - -v /opt/metagpt/workspace:/app/metagpt/workspace \ - metagpt/metagpt:latest - -docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" ``` -The command `docker run ...` do the following things: +detail installation please refer to [docker_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-with-docker) -- Run in privileged mode to have permission to run the browser -- Map host configure file `/opt/metagpt/config/key.yaml` to container `/app/metagpt/config/key.yaml` -- Map host directory `/opt/metagpt/workspace` to container `/app/metagpt/workspace` -- Execute the demo command `python startup.py "Write a cli snake game"` +### QuickStart & Demo Video +- Try it on [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) +- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) +- [Official Demo Video](https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d) -### Build image by yourself +https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 -```bash -# You can also build metagpt image by yourself. -git clone https://github.com/geekan/MetaGPT.git -cd MetaGPT && docker build -t metagpt:custom . -``` +## Tutorial -## Configuration +- 🗒 [Online Document](https://docs.deepwisdom.ai/) +- 💻 [Usage](https://docs.deepwisdom.ai/guide/get_started/quickstart.html) +- 🔎 [What can MetaGPT do?](https://docs.deepwisdom.ai/guide/get_started/introduction.html) +- 🛠 How to build your own agents? + - [MetaGPT Usage & Development Guide | Agent 101](https://docs.deepwisdom.ai/guide/tutorials/agent_101.html) + - [MetaGPT Usage & Development Guide | MultiAgent 101](https://docs.deepwisdom.ai/guide/tutorials/multi_agent_101.html) +- 🧑‍💻 Contribution + - [Develop Roadmap](docs/ROADMAP.md) +- 🔖 Use Cases + - [Debate](https://docs.deepwisdom.ai/guide/use_cases/multi_agent/debate.html) + - [Researcher](https://docs.deepwisdom.ai/guide/use_cases/agent/researcher.html) + - [Recepit Assistant](https://docs.deepwisdom.ai/guide/use_cases/agent/receipt_assistant.html) +- ❓ [FAQs](https://docs.deepwisdom.ai/guide/faq.html) -- Configure your `OPENAI_API_KEY` in any of `config/key.yaml / config/config.yaml / env` -- Priority order: `config/key.yaml > config/config.yaml > env` +## Support -```bash -# Copy the configuration file and make the necessary modifications. -cp config/config.yaml config/key.yaml -``` +### Discard Join US +📢 Join Our [Discord Channel](https://discord.gg/ZRHeExS6xv)! -| Variable Name | config/key.yaml | env | -| ------------------------------------------ | ----------------------------------------- | ----------------------------------------------- | -| OPENAI_API_KEY # Replace with your own key | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | -| OPENAI_API_BASE # Optional | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | +Looking forward to seeing you there! 🎉 -## Tutorial: Initiating a startup - -```shell -# Run the script -python startup.py "Write a cli snake game" -# Do not hire an engineer to implement the project -python startup.py "Write a cli snake game" --implement False -# Hire an engineer and perform code reviews -python startup.py "Write a cli snake game" --code_review True -``` - -After running the script, you can find your new project in the `workspace/` directory. - -### Preference of Platform or Tool - -You can tell which platform or tool you want to use when stating your requirements. - -```shell -python startup.py "Write a cli snake game based on pygame" -``` - -### Usage - -``` -NAME - startup.py - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - -SYNOPSIS - startup.py IDEA - -DESCRIPTION - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - -POSITIONAL ARGUMENTS - IDEA - Type: str - Your innovative idea, such as "Creating a snake game." - -FLAGS - --investment=INVESTMENT - Type: float - Default: 3.0 - As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. - --n_round=N_ROUND - Type: int - Default: 5 - -NOTES - You can also use flags syntax for POSITIONAL ARGUMENTS -``` - -### Code walkthrough - -```python -from metagpt.software_company import SoftwareCompany -from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer - -async def startup(idea: str, investment: float = 3.0, n_round: int = 5): - """Run a startup. Be a boss.""" - company = SoftwareCompany() - company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) -``` - -You can check `examples` for more details on single role (with knowledge base) and LLM only examples. - -## QuickStart - -It is difficult to install and configure the local environment for some users. The following tutorials will allow you to quickly experience the charm of MetaGPT. - -- [MetaGPT quickstart](https://deepwisdom.feishu.cn/wiki/CyY9wdJc4iNqArku3Lncl4v8n2b) - -Try it on Huggingface Space -- https://huggingface.co/spaces/deepwisdom/MetaGPT - -## Citation - -For now, cite the [Arxiv paper](https://arxiv.org/abs/2308.00352): - -```bibtex -@misc{hong2023metagpt, - title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, - author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu}, - year={2023}, - eprint={2308.00352}, - archivePrefix={arXiv}, - primaryClass={cs.AI} -} -``` - -## Contact Information +### Contact Information If you have any questions or feedback about this project, please feel free to contact us. We highly appreciate your suggestions! @@ -317,13 +122,17 @@ ## Contact Information We will respond to all questions within 2-3 business days. -## Demo +## Citation -https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d +For now, cite the [arXiv paper](https://arxiv.org/abs/2308.00352): -## Join us - -📢 Join Our Discord Channel! -https://discord.gg/ZRHeExS6xv - -Looking forward to seeing you there! 🎉 +```bibtex +@misc{hong2023metagpt, + title={MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework}, + author={Sirui Hong and Mingchen Zhuge and Jonathan Chen and Xiawu Zheng and Yuheng Cheng and Ceyao Zhang and Jinlin Wang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu and Jürgen Schmidhuber}, + year={2023}, + eprint={2308.00352}, + archivePrefix={arXiv}, + primaryClass={cs.AI} +} +``` diff --git a/config/config.yaml b/config/config.yaml index b2c50991d..bed67083c 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,7 +7,7 @@ ## Or, you can configure OPENAI_PROXY to access official OPENAI_API_BASE. OPENAI_API_BASE: "https://api.openai.com/v1" #OPENAI_PROXY: "http://127.0.0.1:8118" -#OPENAI_API_KEY: "YOUR_API_KEY" +#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model OPENAI_API_MODEL: "gpt-4" MAX_TOKENS: 1500 RPM: 10 @@ -31,6 +31,9 @@ RPM: 10 #DEPLOYMENT_NAME: "YOUR_DEPLOYMENT_NAME" #DEPLOYMENT_ID: "YOUR_DEPLOYMENT_ID" +#### if zhipuai from `https://open.bigmodel.cn`. You can set here or export API_KEY="YOUR_API_KEY" +# ZHIPUAI_API_KEY: "YOUR_API_KEY" + #### for Search ## Supported values: serpapi/google/serper/ddg diff --git a/docs/README_CN.md b/docs/README_CN.md index 9d6f34c11..038925184 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -12,14 +12,13 @@ # MetaGPT: 多智能体框架 CN doc EN doc JA doc -Discord Follow +Discord Follow License: MIT roadmap Twitter Follow

- AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces Hugging Face @@ -33,57 +32,35 @@ # MetaGPT: 多智能体框架

软件公司多角色示意图（正在逐步实现）

-## MetaGPT 的能力 - -https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 - - -## 示例（均由 GPT-4 生成） - -例如，键入`python startup.py "写个类似今日头条的推荐系统"`并回车，你会获得一系列输出，其一是数据结构与API设计 - -![今日头条 Recsys 数据 & API 设计](resources/workspace/content_rec_sys/resources/data_api_design.png) - -这需要大约**0.2美元**（GPT-4 API的费用）来生成一个带有分析和设计的示例，大约2.0美元用于一个完整的项目 - ## 安装 - -### 传统安装 +### Pip安装 ```bash -# 第 1 步：确保您的系统上安装了 NPM。并使用npm安装mermaid-js +# 第 1 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： +# 可以使用conda来初始化新的python环境 +# conda create -n metagpt python=3.9 +# conda activate metagpt +python3 --version + +# 第 2 步：克隆最新仓库到您的本地机器，并进行安装。 +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT +pip3 install -e. # 或者 pip3 install metagpt # 安装稳定版本 + +# 第 3 步：执行startup.py +# 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY +python3 startup.py "Write a cli snake game" + +# 第 4 步【可选的】：如果你想在执行过程中保存像象限图、系统设计、序列流程等图表这些产物，可以在第3步前执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。 +# 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid-js npm --version sudo npm install -g @mermaid-js/mermaid-cli - -# 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查： -python --version - -# 第 3 步：克隆仓库到您的本地机器，并进行安装。 -git clone https://github.com/geekan/metagpt -cd metagpt -pip install -e. ``` -**注意：** - -- 如果已经安装了Chrome、Chromium或MS Edge，可以通过将环境变量`PUPPETEER_SKIP_CHROMIUM_DOWNLOAD`设置为`true`来跳过下载Chromium。 - -- 一些人在全局安装此工具时遇到问题。在本地安装是替代解决方案， - - ```bash - npm install @mermaid-js/mermaid-cli - ``` - -- 不要忘记在config.yml中为mmdc配置配置， - - ```yml - PUPPETEER_CONFIG: "./config/puppeteer-config.json" - MMDC: "./node_modules/.bin/mmdc" - ``` - -- 如果`pip install -e.`失败并显示错误`[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`，请尝试使用`pip install -e. --user`运行。 +详细的安装请安装 [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version) ### Docker安装 +> 注意：在Windows中，你需要将 "/opt/metagpt" 替换为Docker具有创建权限的目录，比如"D:\Users\x\metagpt" ```bash # 步骤1: 下载metagpt官方镜像并准备好config.yaml @@ -99,121 +76,41 @@ # 步骤2: 使用容器运行metagpt演示 -v /opt/metagpt/workspace:/app/metagpt/workspace \ metagpt/metagpt:latest \ python startup.py "Write a cli snake game" - -# 您也可以启动一个容器并在其中执行命令 -docker run --name metagpt -d \ - --privileged \ - -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \ - -v /opt/metagpt/workspace:/app/metagpt/workspace \ - metagpt/metagpt:latest - -docker exec -it metagpt /bin/bash -$ python startup.py "Write a cli snake game" ``` -`docker run ...`做了以下事情: +详细的安装请安装 [docker_install](https://docs.deepwisdom.ai/zhcn/guide/get_started/installation.html#%E4%BD%BF%E7%94%A8docker%E5%AE%89%E8%A3%85) -- 以特权模式运行，有权限运行浏览器 -- 将主机文件 `/opt/metagpt/config/key.yaml` 映射到容器文件 `/app/metagpt/config/key.yaml` -- 将主机目录 `/opt/metagpt/workspace` 映射到容器目录 `/app/metagpt/workspace` -- 执行示例命令 `python startup.py "Write a cli snake game"` +### 快速开始的演示视频 +- 在 [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) 上进行体验 +- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) +- [官方演示视频](https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d) -### 自己构建镜像 +https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 -```bash -# 您也可以自己构建metagpt镜像 -git clone https://github.com/geekan/MetaGPT.git -cd MetaGPT && docker build -t metagpt:custom . -``` +## 教程 +- 🗒 [在线文档](https://docs.deepwisdom.ai/zhcn/) +- 💻 [如何使用](https://docs.deepwisdom.ai/zhcn/guide/get_started/quickstart.html) +- 🔎 [MetaGPT的能力及应用场景](https://docs.deepwisdom.ai/zhcn/guide/get_started/introduction.html) +- 🛠 如何构建你自己的智能体？ + - [MetaGPT的使用和开发教程 | 智能体入门](https://docs.deepwisdom.ai/zhcn/guide/tutorials/agent_101.html) + - [MetaGPT的使用和开发教程 | 多智能体入门](https://docs.deepwisdom.ai/zhcn/guide/tutorials/multi_agent_101.html) +- 🧑‍💻 贡献 + - [开发路线图](ROADMAP.md) +- 🔖 示例 + - [辩论](https://docs.deepwisdom.ai/zhcn/guide/use_cases/multi_agent/debate.html) + - [调研员](https://docs.deepwisdom.ai/zhcn/guide/use_cases/agent/researcher.html) + - [票据助手](https://docs.deepwisdom.ai/zhcn/guide/use_cases/agent/receipt_assistant.html) +- ❓ [常见问题解答](https://docs.deepwisdom.ai/zhcn/guide/faq.html) -## 配置 +## 支持 -- 在 `config/key.yaml / config/config.yaml / env` 中配置您的 `OPENAI_API_KEY` -- 优先级顺序：`config/key.yaml > config/config.yaml > env` +### 加入我们 -```bash -# 复制配置文件并进行必要的修改 -cp config/config.yaml config/key.yaml -``` +📢 加入我们的[Discord频道](https://discord.gg/ZRHeExS6xv)！ -| 变量名 | config/key.yaml | env | -| ----------------------------------- | ----------------------------------------- | ----------------------------------------------- | -| OPENAI_API_KEY # 用您自己的密钥替换 | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | -| OPENAI_API_BASE # 可选 | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | +期待在那里与您相见！🎉 -## 示例：启动一个创业公司 - -```shell -python startup.py "写一个命令行贪吃蛇" -# 开启code review模式会花费更多的金钱, 但是会提升代码质量和成功率 -python startup.py "写一个命令行贪吃蛇" --code_review True -``` - -运行脚本后，您可以在 `workspace/` 目录中找到您的新项目。 -### 平台或工具的倾向性 -可以在阐述需求时说明想要使用的平台或工具。 -例如： -```shell -python startup.py "写一个基于pygame的命令行贪吃蛇" -``` - -### 使用 - -``` -名称 - startup.py - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 - -概要 - startup.py IDEA - -描述 - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。 - -位置参数 - IDEA - 类型: str - 您的创新想法，例如"写一个命令行贪吃蛇。" - -标志 - --investment=INVESTMENT - 类型: float - 默认值: 3.0 - 作为投资者，您有机会向这家AI公司投入一定的美元金额。 - --n_round=N_ROUND - 类型: int - 默认值: 5 - -备注 - 您也可以用`标志`的语法，来处理`位置参数` -``` - -### 代码实现 - -```python -from metagpt.software_company import SoftwareCompany -from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer - -async def startup(idea: str, investment: float = 3.0, n_round: int = 5): - """运行一个创业公司。做一个老板""" - company = SoftwareCompany() - company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) -``` - -你可以查看`examples`，其中有单角色（带知识库）的使用例子与仅LLM的使用例子。 - -## 快速体验 -对一些用户来说，安装配置本地环境是有困难的，下面这些教程能够让你快速体验到MetaGPT的魅力。 - -- [MetaGPT快速体验](https://deepwisdom.feishu.cn/wiki/Q8ycw6J9tiNXdHk66MRcIN8Pnlg) - -可直接在Huggingface Space体验 - -- https://huggingface.co/spaces/deepwisdom/MetaGPT - -## 联系信息 +### 联系信息 如果您对这个项目有任何问题或反馈，欢迎联系我们。我们非常欢迎您的建议！ @@ -222,13 +119,17 @@ ## 联系信息 我们会在2-3个工作日内回复所有问题。 -## 演示 +## 引用 -https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d +引用 [arXiv paper](https://arxiv.org/abs/2308.00352): -## 加入我们 - -📢 加入我们的Discord频道！ -https://discord.gg/ZRHeExS6xv - -期待在那里与您相见！🎉 +```bibtex +@misc{hong2023metagpt, + title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, + author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu}, + year={2023}, + eprint={2308.00352}, + archivePrefix={arXiv}, + primaryClass={cs.AI} +} +``` diff --git a/docs/README_JA.md b/docs/README_JA.md index 2b2c35a62..411d190b4 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -19,7 +19,6 @@ # MetaGPT: マルチエージェントフレームワーク

- AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces Hugging Face @@ -60,17 +59,22 @@ ### インストールビデオガイド ### 伝統的なインストール ```bash -# ステップ 1: NPM がシステムにインストールされていることを確認してください。次に mermaid-js をインストールします。(お使いのコンピューターに npm がない場合は、Node.js 公式サイトで Node.js https://nodejs.org/ をインストールしてください。） -npm --version -sudo npm install -g @mermaid-js/mermaid-cli - -# ステップ 2: Python 3.9+ がシステムにインストールされていることを確認してください。これを確認するには: +# ステップ 1: Python 3.9+ がシステムにインストールされていることを確認してください。これを確認するには: python --version -# ステップ 3: リポジトリをローカルマシンにクローンし、インストールする。 -git clone https://github.com/geekan/metagpt -cd metagpt +# ステップ 2: リポジトリをローカルマシンにクローンし、インストールする。 +git clone https://github.com/geekan/MetaGPT.git +cd MetaGPT pip install -e. + +# ステップ 3: startup.py を実行する +# config.yaml を key.yaml にコピーし、独自の OPENAI_API_KEY を設定します +python3 startup.py "Write a cli snake game" + +# ステップ 4 [オプション]: 実行中に PRD ファイルなどのアーティファクトを保存する場合は、ステップ 3 の前にこのステップを実行できます。デフォルトでは、フレームワークには互換性があり、この手順を実行しなくてもプロセス全体を完了できます。 +# NPM がシステムにインストールされていることを確認してください。次に mermaid-js をインストールします。(お使いのコンピューターに npm がない場合は、Node.js 公式サイトで Node.js https://nodejs.org/ をインストールしてください。） +npm --version +sudo npm install -g @mermaid-js/mermaid-cli ``` **注:** @@ -159,6 +163,7 @@ # ステップ 3: リポジトリをローカルマシンにクローンし、 注: この方法は pdf エクスポートに対応していません。 ### Docker によるインストール +> Windowsでは、"/opt/metagpt"をDockerが作成する権限を持つディレクトリに置き換える必要があります。例えば、"D:\Users\x\metagpt"などです。 ```bash # ステップ 1: metagpt 公式イメージをダウンロードし、config.yaml を準備する @@ -270,12 +275,12 @@ ### 使用方法 ### コードウォークスルー ```python -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.roles import ProjectManager, ProductManager, Architect, Engineer async def startup(idea: str, investment: float = 3.0, n_round: int = 5): """スタートアップを実行する。ボスになる。""" - company = SoftwareCompany() + company = Team() company.hire([ProductManager(), Architect(), ProjectManager(), Engineer()]) company.invest(investment) company.start_project(idea) @@ -295,12 +300,12 @@ ## クイックスタート ## 引用 -現時点では、[Arxiv 論文](https://arxiv.org/abs/2308.00352)を引用してください: +現時点では、[arXiv 論文](https://arxiv.org/abs/2308.00352)を引用してください: ```bibtex @misc{hong2023metagpt, - title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, - author={Sirui Hong and Xiawu Zheng and Jonathan Chen and Yuheng Cheng and Jinlin Wang and Ceyao Zhang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu}, + title={MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework}, + author={Sirui Hong and Mingchen Zhuge and Jonathan Chen and Xiawu Zheng and Yuheng Cheng and Ceyao Zhang and Jinlin Wang and Zili Wang and Steven Ka Shing Yau and Zijuan Lin and Liyang Zhou and Chenyu Ran and Lingfeng Xiao and Chenglin Wu and Jürgen Schmidhuber}, year={2023}, eprint={2308.00352}, archivePrefix={arXiv}, diff --git a/examples/build_customized_agent.py b/examples/build_customized_agent.py index 87d7a9c76..be34e5e5e 100644 --- a/examples/build_customized_agent.py +++ b/examples/build_customized_agent.py @@ -9,6 +9,7 @@ import asyncio import fire +from metagpt.llm import LLM from metagpt.actions import Action from metagpt.roles import Role from metagpt.schema import Message @@ -19,19 +20,10 @@ class SimpleWriteCode(Action): PROMPT_TEMPLATE = """ Write a python function that can {instruction} and provide two runnnable test cases. Return ```python your_code_here ``` with NO other texts, - example: - ```python - # function - def add(a, b): - return a + b - # test cases - print(add(1, 2)) - print(add(3, 4)) - ``` your code: """ - def __init__(self, name="SimpleWriteCode", context=None, llm=None): + def __init__(self, name: str = "SimpleWriteCode", context=None, llm: LLM = None): super().__init__(name, context, llm) async def run(self, instruction: str): @@ -51,8 +43,9 @@ class SimpleWriteCode(Action): code_text = match.group(1) if match else rsp return code_text + class SimpleRunCode(Action): - def __init__(self, name="SimpleRunCode", context=None, llm=None): + def __init__(self, name: str = "SimpleRunCode", context=None, llm: LLM = None): super().__init__(name, context, llm) async def run(self, code_text: str): @@ -61,6 +54,7 @@ class SimpleRunCode(Action): logger.info(f"{code_result=}") return code_result + class SimpleCoder(Role): def __init__( self, @@ -73,16 +67,16 @@ class SimpleCoder(Role): async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") - todo = self._rc.todo + todo = self._rc.todo # todo will be SimpleWriteCode() - msg = self._rc.memory.get()[-1] # retrieve the latest memory - instruction = msg.content + msg = self.get_memories(k=1)[0] # find the most recent messages - code_text = await SimpleWriteCode().run(instruction) - msg = Message(content=code_text, role=self.profile, cause_by=todo) + code_text = await todo.run(msg.content) + msg = Message(content=code_text, role=self.profile, cause_by=type(todo)) return msg + class RunnableCoder(Role): def __init__( self, @@ -92,43 +86,23 @@ class RunnableCoder(Role): ): super().__init__(name, profile, **kwargs) self._init_actions([SimpleWriteCode, SimpleRunCode]) - - async def _think(self) -> None: - if self._rc.todo is None: - self._set_state(0) - return - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None + self._set_react_mode(react_mode="by_order") async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") + # By choosing the Action by order under the hood + # todo will be first SimpleWriteCode() then SimpleRunCode() todo = self._rc.todo - msg = self._rc.memory.get()[-1] - if isinstance(todo, SimpleWriteCode): - instruction = msg.content - result = await SimpleWriteCode().run(instruction) + msg = self.get_memories(k=1)[0] # find the most k recent messages + result = await todo.run(msg.content) - elif isinstance(todo, SimpleRunCode): - code_text = msg.content - result = await SimpleRunCode().run(code_text) - - msg = Message(content=result, role=self.profile, cause_by=todo) + msg = Message(content=result, role=self.profile, cause_by=type(todo)) self._rc.memory.add(msg) return msg - async def _react(self) -> Message: - while True: - await self._think() - if self._rc.todo is None: - break - await self._act() - return Message(content="All job done", role=self.profile) -def main(msg="write a function that calculates the sum of a list"): +def main(msg="write a function that calculates the product of a list and run it"): # role = SimpleCoder() role = RunnableCoder() logger.info(msg) diff --git a/examples/debate.py b/examples/debate.py index 05db28070..a37e60848 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -7,14 +7,14 @@ import asyncio import platform import fire -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.actions import Action, BossRequirement from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger -class ShoutOut(Action): - """Action: Shout out loudly in a debate (quarrel)""" +class SpeakAloud(Action): + """Action: Speak out aloud in a debate (quarrel)""" PROMPT_TEMPLATE = """ ## BACKGROUND @@ -27,7 +27,7 @@ class ShoutOut(Action): craft a strong and emotional response in 80 words, in {name}'s rhetoric and viewpoints, your will argue: """ - def __init__(self, name="ShoutOut", context=None, llm=None): + def __init__(self, name="SpeakAloud", context=None, llm=None): super().__init__(name, context, llm) async def run(self, context: str, name: str, opponent_name: str): @@ -39,96 +39,57 @@ class ShoutOut(Action): return rsp -class Trump(Role): +class Debator(Role): def __init__( self, - name: str = "Trump", - profile: str = "Republican", + name: str, + profile: str, + opponent_name: str, **kwargs, ): super().__init__(name, profile, **kwargs) - self._init_actions([ShoutOut]) - self._watch([ShoutOut]) - self.name = "Trump" - self.opponent_name = "Biden" + self._init_actions([SpeakAloud]) + self._watch([BossRequirement, SpeakAloud]) + self.name = name + self.opponent_name = opponent_name async def _observe(self) -> int: await super()._observe() # accept messages sent (from opponent) to self, disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] + self._rc.news = [msg for msg in self._rc.news if msg.send_to == self.name] return len(self._rc.news) async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") + todo = self._rc.todo # An instance of SpeakAloud - msg_history = self._rc.memory.get_by_actions([ShoutOut]) - context = [] - for m in msg_history: - context.append(str(m)) - context = "\n".join(context) + memories = self.get_memories() + context = "\n".join(f"{msg.sent_from}: {msg.content}" for msg in memories) + # print(context) - rsp = await ShoutOut().run(context=context, name=self.name, opponent_name=self.opponent_name) + rsp = await todo.run(context=context, name=self.name, opponent_name=self.opponent_name) msg = Message( content=rsp, role=self.profile, - cause_by=ShoutOut, + cause_by=type(todo), sent_from=self.name, send_to=self.opponent_name, ) - return msg - -class Biden(Role): - def __init__( - self, - name: str = "Biden", - profile: str = "Democrat", - **kwargs, - ): - super().__init__(name, profile, **kwargs) - self._init_actions([ShoutOut]) - self._watch([BossRequirement, ShoutOut]) - self.name = "Biden" - self.opponent_name = "Trump" - - async def _observe(self) -> int: - await super()._observe() - # accept the very first human instruction (the debate topic) or messages sent (from opponent) to self, - # disregard own messages from the last round - self._rc.news = [msg for msg in self._rc.news if msg.cause_by == BossRequirement or msg.send_to == self.name] - return len(self._rc.news) - - async def _act(self) -> Message: - logger.info(f"{self._setting}: ready to {self._rc.todo}") - - msg_history = self._rc.memory.get_by_actions([BossRequirement, ShoutOut]) - context = [] - for m in msg_history: - context.append(str(m)) - context = "\n".join(context) - - rsp = await ShoutOut().run(context=context, name=self.name, opponent_name=self.opponent_name) - - msg = Message( - content=rsp, - role=self.profile, - cause_by=ShoutOut, - sent_from=self.name, - send_to=self.opponent_name, - ) + self._rc.memory.add(msg) return msg -async def startup(idea: str, investment: float = 3.0, n_round: int = 5, - code_review: bool = False, run_tests: bool = False): - """We reuse the startup paradigm for roles to interact with each other. - Now we run a startup of presidents and watch they quarrel. :) """ - company = SoftwareCompany() - company.hire([Biden(), Trump()]) - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) +async def debate(idea: str, investment: float = 3.0, n_round: int = 5): + """Run a team of presidents and watch they quarrel. :) """ + Biden = Debator(name="Biden", profile="Democrat", opponent_name="Trump") + Trump = Debator(name="Trump", profile="Republican", opponent_name="Biden") + team = Team() + team.hire([Biden, Trump]) + team.invest(investment) + team.start_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first + await team.run(n_round=n_round) def main(idea: str, investment: float = 3.0, n_round: int = 10): @@ -141,7 +102,7 @@ def main(idea: str, investment: float = 3.0, n_round: int = 10): """ if platform.system() == "Windows": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - asyncio.run(startup(idea, investment, n_round)) + asyncio.run(debate(idea, investment, n_round)) if __name__ == '__main__': diff --git a/metagpt/config.py b/metagpt/config.py index 27455d38d..3f9e742bd 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -45,10 +45,11 @@ class Config(metaclass=Singleton): self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") - if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key - ): - raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") + self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY") + if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and \ + (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) and \ + (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key): + raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy if openai_proxy: diff --git a/metagpt/const.py b/metagpt/const.py index 7f3f87dfa..407ce803a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -6,7 +6,7 @@ @File : const.py """ from pathlib import Path - +from loguru import logger def get_project_root(): """Search upwards to find the project root directory.""" @@ -17,10 +17,15 @@ def get_project_root(): or (current_path / ".project_root").exists() or (current_path / ".gitignore").exists() ): + # use metagpt with git clone will land here + logger.info(f"PROJECT_ROOT set to {str(current_path)}") return current_path parent_path = current_path.parent if parent_path == current_path: - raise Exception("Project root not found.") + # use metagpt with pip install will land here + cwd = Path.cwd() + logger.info(f"PROJECT_ROOT set to current working directory: {str(cwd)}") + return cwd current_path = parent_path diff --git a/metagpt/llm.py b/metagpt/llm.py index e6f815950..4edcd7a83 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -6,14 +6,27 @@ @File : llm.py """ +from metagpt.logs import logger +from metagpt.config import CONFIG from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM +from metagpt.provider.openai_api import OpenAIGPTAPI +from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +from metagpt.provider.spark_api import SparkAPI +from metagpt.provider.human_provider import HumanProvider -DEFAULT_LLM = LLM() -CLAUDE_LLM = Claude() -async def ai_func(prompt): - """使用LLM进行QA - QA with LLMs - """ - return await DEFAULT_LLM.aask(prompt) +def LLM() -> "BaseGPTAPI": + """ initialize different LLM instance according to the key field existence""" + # TODO a little trick, can use registry to initialize LLM instance further + if CONFIG.openai_api_key: + llm = OpenAIGPTAPI() + elif CONFIG.claude_api_key: + llm = Claude() + elif CONFIG.spark_api_key: + llm = SparkAPI() + elif CONFIG.zhipuai_api_key: + llm = ZhiPuAIGPTAPI() + else: + raise RuntimeError("You should config a LLM configuration first") + + return llm diff --git a/metagpt/provider/base_chatbot.py b/metagpt/provider/base_chatbot.py index abdf423f4..72e6c94f9 100644 --- a/metagpt/provider/base_chatbot.py +++ b/metagpt/provider/base_chatbot.py @@ -13,6 +13,7 @@ from dataclasses import dataclass class BaseChatbot(ABC): """Abstract GPT class""" mode: str = "API" + use_system_prompt: bool = True @abstractmethod def ask(self, msg: str) -> str: diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index de61167b9..b6b034329 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : base_gpt_api.py """ +import json from abc import abstractmethod from typing import Optional @@ -14,7 +15,8 @@ from metagpt.provider.base_chatbot import BaseChatbot class BaseGPTAPI(BaseChatbot): """GPT API abstract class, requiring all inheritors to provide a series of standard capabilities""" - system_prompt = 'You are a helpful assistant.' + + system_prompt = "You are a helpful assistant." def _user_msg(self, msg: str) -> dict[str, str]: return {"role": "user", "content": msg} @@ -32,15 +34,17 @@ class BaseGPTAPI(BaseChatbot): return self._system_msg(self.system_prompt) def ask(self, msg: str) -> str: - message = [self._default_system_msg(), self._user_msg(msg)] + message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)] rsp = self.completion(message) return self.get_choice_text(rsp) async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: if system_msgs: - message = self._system_msgs(system_msgs) + [self._user_msg(msg)] + message = self._system_msgs(system_msgs) + [self._user_msg(msg)] if self.use_system_prompt \ + else [self._user_msg(msg)] else: - message = [self._default_system_msg(), self._user_msg(msg)] + message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt \ + else [self._user_msg(msg)] rsp = await self.acompletion_text(message, stream=True) logger.debug(message) # logger.debug(rsp) @@ -108,11 +112,50 @@ class BaseGPTAPI(BaseChatbot): """Required to provide the first text of choice""" return rsp.get("choices")[0]["message"]["content"] + def get_choice_function(self, rsp: dict) -> dict: + """Required to provide the first function of choice + :param dict rsp: OpenAI chat.comletion respond JSON, Note "message" must include "tool_calls", + and "tool_calls" must include "function", for example: + {... + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_Y5r6Ddr2Qc2ZrqgfwzPX5l72", + "type": "function", + "function": { + "name": "execute", + "arguments": "{\n \"language\": \"python\",\n \"code\": \"print('Hello, World!')\"\n}" + } + } + ] + }, + "finish_reason": "stop" + } + ], + ...} + :return dict: return first function of choice, for exmaple, + {'name': 'execute', 'arguments': '{\n "language": "python",\n "code": "print(\'Hello, World!\')"\n}'} + """ + return rsp.get("choices")[0]["message"]["tool_calls"][0]["function"].to_dict() + + def get_choice_function_arguments(self, rsp: dict) -> dict: + """Required to provide the first function arguments of choice. + + :param dict rsp: same as in self.get_choice_function(rsp) + :return dict: return the first function arguments of choice, for example, + {'language': 'python', 'code': "print('Hello, World!')"} + """ + return json.loads(self.get_choice_function(rsp)["arguments"]) + def messages_to_prompt(self, messages: list[dict]): """[{"role": "user", "content": msg}] to user: etc.""" - return '\n'.join([f"{i['role']}: {i['content']}" for i in messages]) + return "\n".join([f"{i['role']}: {i['content']}" for i in messages]) def messages_to_dict(self, messages): """objects to [{"role": "user", "content": msg}] etc.""" return [i.to_dict() for i in messages] - \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 6ebed2c16..34e5693f8 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -21,6 +21,8 @@ from tenacity import ( from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE +from metagpt.schema import Message from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( TOKEN_COSTS, @@ -110,7 +112,6 @@ class CostManager(metaclass=Singleton): """ return self.total_completion_tokens - def get_total_cost(self): """ Get the total cost of API calls. @@ -120,7 +121,6 @@ class CostManager(metaclass=Singleton): """ return self.total_cost - def get_costs(self) -> Costs: """Get all costs""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) @@ -181,7 +181,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self._update_costs(usage) return full_reply_content - def _cons_kwargs(self, messages: list[dict]) -> dict: + def _cons_kwargs(self, messages: list[dict], **configs) -> dict: kwargs = { "messages": messages, "max_tokens": self.get_max_tokens(messages), @@ -190,6 +190,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): "temperature": 0.3, "timeout": 3, } + if configs: + kwargs.update(configs) + if CONFIG.openai_api_type == "azure": if CONFIG.deployment_name and CONFIG.deployment_id: raise ValueError("You can only use one of the `deployment_id` or `deployment_name` model") @@ -239,6 +242,81 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): rsp = await self._achat_completion(messages) return self.get_choice_text(rsp) + def _func_configs(self, messages: list[dict], **kwargs) -> dict: + """ + Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create + """ + if "tools" not in kwargs: + configs = { + "tools": [{"type": "function", "function": GENERAL_FUNCTION_SCHEMA}], + "tool_choice": GENERAL_TOOL_CHOICE, + } + kwargs.update(configs) + + return self._cons_kwargs(messages, **kwargs) + + def _chat_completion_function(self, messages: list[dict], **kwargs) -> dict: + rsp = self.llm.ChatCompletion.create(**self._func_configs(messages, **kwargs)) + self._update_costs(rsp.get("usage")) + return rsp + + async def _achat_completion_function(self, messages: list[dict], **chat_configs) -> dict: + rsp = await self.llm.ChatCompletion.acreate(**self._func_configs(messages, **chat_configs)) + self._update_costs(rsp.get("usage")) + return rsp + + def _process_message(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]: + """convert messages to list[dict].""" + if isinstance(messages, list): + messages = [Message(msg) if isinstance(msg, str) else msg for msg in messages] + return [msg if isinstance(msg, dict) else msg.to_dict() for msg in messages] + + if isinstance(messages, Message): + messages = [messages.to_dict()] + elif isinstance(messages, str): + messages = [{"role": "user", "content": messages}] + else: + raise ValueError( + f"Only support messages type are: str, Message, list[dict], but got {type(messages).__name__}!" + ) + return messages + + def ask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: + """Use function of tools to ask a code. + + Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create + + Examples: + + >>> llm = OpenAIGPTAPI() + >>> llm.ask_code("Write a python hello world code.") + {'language': 'python', 'code': "print('Hello, World!')"} + >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}] + >>> llm.ask_code(msg) + {'language': 'python', 'code': "print('Hello, World!')"} + """ + messages = self._process_message(messages) + rsp = self._chat_completion_function(messages, **kwargs) + return self.get_choice_function_arguments(rsp) + + async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict: + """Use function of tools to ask a code. + + Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create + + Examples: + + >>> llm = OpenAIGPTAPI() + >>> rsp = await llm.ask_code("Write a python hello world code.") + >>> rsp + {'language': 'python', 'code': "print('Hello, World!')"} + >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}] + >>> rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"} + """ + messages = self._process_message(messages) + rsp = await self._achat_completion_function(messages, **kwargs) + return self.get_choice_function_arguments(rsp) + def _calc_usage(self, messages: list[dict], rsp: str) -> dict: usage = {} if CONFIG.calc_usage: diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 6d65575a8..1f6685b38 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -207,6 +207,7 @@ class Engineer(Role): async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" + logger.info(f"{self._setting}: ready to WriteCode") if self.use_code_review: return await self._act_sp_precision() return await self._act_sp() diff --git a/metagpt/roles/invoice_ocr_assistant.py b/metagpt/roles/invoice_ocr_assistant.py index c307b20c0..15f831c97 100644 --- a/metagpt/roles/invoice_ocr_assistant.py +++ b/metagpt/roles/invoice_ocr_assistant.py @@ -42,17 +42,7 @@ class InvoiceOCRAssistant(Role): self.filename = "" self.origin_query = "" self.orc_data = None - - async def _think(self) -> None: - """Determine the next action to be taken by the role.""" - if self._rc.todo is None: - self._set_state(0) - return - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None + self._set_react_mode(react_mode="by_order") async def _act(self) -> Message: """Perform an action as determined by the role. @@ -94,17 +84,3 @@ class InvoiceOCRAssistant(Role): msg = Message(content=content, instruct_content=resp) self._rc.memory.add(msg) return msg - - async def _react(self) -> Message: - """Execute the invoice ocr assistant's think and actions. - - Returns: - A message containing the final result of the assistant's actions. - """ - while True: - await self._think() - if self._rc.todo is None: - break - msg = await self._act() - return msg - diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index acb46c718..c5512121a 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -31,20 +31,11 @@ class Researcher(Role): ): super().__init__(name, profile, goal, constraints, **kwargs) self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)]) + self._set_react_mode(react_mode="by_order") self.language = language if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") - async def _think(self) -> None: - if self._rc.todo is None: - self._set_state(0) - return - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None - async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") todo = self._rc.todo @@ -73,12 +64,8 @@ class Researcher(Role): self._rc.memory.add(ret) return ret - async def _react(self) -> Message: - while True: - await self._think() - if self._rc.todo is None: - break - msg = await self._act() + async def react(self) -> Message: + msg = await super().react() report = msg.instruct_content self.write_report(report.topic, report.content) return msg diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 44bb3e976..b96c361c0 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -7,14 +7,15 @@ """ from __future__ import annotations -from typing import Iterable, Type +from typing import Iterable, Type, Union +from enum import Enum from pydantic import BaseModel, Field # from metagpt.environment import Environment from metagpt.config import CONFIG from metagpt.actions import Action, ActionOutput -from metagpt.llm import LLM +from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory from metagpt.schema import Message @@ -27,12 +28,14 @@ Please note that only the text between the first and second "===" is information {history} === -You can now choose one of the following stages to decide the stage you need to go in the next step: +Your previous stage: {previous_state} + +Now choose one of the following stages you need to go to in the next step: {states} Just answer a number between 0-{n_states}, choose the most suitable stage according to the understanding of the conversation. Please note that the answer only needs a number, no need to add any other text. -If there is no conversation record, choose 0. +If you think you have completed your goal and don't need to go to any of the stages, return -1. Do not answer anything else, and do not add any other information in your answer. """ @@ -46,6 +49,14 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi {name}: {result} """ +class RoleReactMode(str, Enum): + REACT = "react" + BY_ORDER = "by_order" + PLAN_AND_ACT = "plan_and_act" + + @classmethod + def values(cls): + return [item.value for item in cls] class RoleSetting(BaseModel): """Role Settings""" @@ -54,6 +65,7 @@ class RoleSetting(BaseModel): goal: str constraints: str desc: str + is_human: bool def __str__(self): return f"{self.name}({self.profile})" @@ -67,10 +79,12 @@ class RoleContext(BaseModel): env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) - state: int = Field(default=0) + state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) + react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes + max_react_loop: int = 1 class Config: arbitrary_types_allowed = True @@ -93,9 +107,10 @@ class RoleContext(BaseModel): class Role: """Role/Agent""" - def __init__(self, name="", profile="", goal="", constraints="", desc=""): - self._llm = LLM() - self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) + def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False): + self._llm = LLM() if not is_human else HumanProvider() + self._setting = RoleSetting(name=name, profile=profile, goal=goal, + constraints=constraints, desc=desc, is_human=is_human) self._states = [] self._actions = [] self._role_id = str(self._setting) @@ -109,24 +124,48 @@ class Role: self._reset() for idx, action in enumerate(actions): if not isinstance(action, Action): - i = action("") + i = action("", llm=self._llm) else: + if self._setting.is_human and not isinstance(action.llm, HumanProvider): + logger.warning(f"is_human attribute does not take effect," + f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") i = action i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) self._states.append(f"{idx}. {action}") + def _set_react_mode(self, react_mode: str, max_react_loop: int = 1): + """Set strategy of the Role reacting to observed Message. Variation lies in how + this Role elects action to perform during the _think stage, especially if it is capable of multiple Actions. + + Args: + react_mode (str): Mode for choosing action during the _think stage, can be one of: + "react": standard think-act loop in the ReAct paper, alternating thinking and acting to solve the task, i.e. _think -> _act -> _think -> _act -> ... + Use llm to select actions in _think dynamically; + "by_order": switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...; + "plan_and_act": first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... + Use llm to come up with the plan dynamically. + Defaults to "react". + max_react_loop (int): Maximum react cycles to execute, used to prevent the agent from reacting forever. + Take effect only when react_mode is react, in which we use llm to choose actions, including termination. + Defaults to 1, i.e. _think -> _act (-> return result and end) + """ + assert react_mode in RoleReactMode.values(), f"react_mode must be one of {RoleReactMode.values()}" + self._rc.react_mode = react_mode + if react_mode == RoleReactMode.REACT: + self._rc.max_react_loop = max_react_loop + def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors""" self._rc.watch.update(actions) # check RoleContext after adding watch actions self._rc.check(self._role_id) - def _set_state(self, state): + def _set_state(self, state: int): """Update the current state.""" self._rc.state = state logger.debug(self._actions) - self._rc.todo = self._actions[self._rc.state] + self._rc.todo = self._actions[self._rc.state] if state >= 0 else None def set_env(self, env: 'Environment'): """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" @@ -151,13 +190,19 @@ class Role: return prompt = self._get_prefix() prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1) + n_states=len(self._states) - 1, previous_state=self._rc.state) + # print(prompt) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") - if not next_state.isdigit() or int(next_state) not in range(len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}') - next_state = "0" - self._set_state(int(next_state)) + if (not next_state.isdigit() and next_state != "-1") \ + or int(next_state) not in range(-1, len(self._states)): + logger.warning(f'Invalid answer of state, {next_state=}, will be set to -1') + next_state = -1 + else: + next_state = int(next_state) + if next_state == -1: + logger.info(f"End actions with {next_state=}") + self._set_state(next_state) async def _act(self) -> Message: # prompt = self.get_prefix() @@ -203,10 +248,45 @@ class Role: self._rc.env.publish_message(msg) async def _react(self) -> Message: - """Think first, then act""" - await self._think() - logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") - return await self._act() + """Think first, then act, until the Role _think it is time to stop and requires no more todo. + This is the standard think-act loop in the ReAct paper, which alternates thinking and acting in task solving, i.e. _think -> _act -> _think -> _act -> ... + Use llm to select actions in _think dynamically + """ + actions_taken = 0 + rsp = Message("No actions taken yet") # will be overwritten after Role _act + while actions_taken < self._rc.max_react_loop: + # think + await self._think() + if self._rc.todo is None: + break + # act + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + rsp = await self._act() + actions_taken += 1 + return rsp # return output from the last action + + async def _act_by_order(self) -> Message: + """switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...""" + for i in range(len(self._states)): + self._set_state(i) + rsp = await self._act() + return rsp # return output from the last action + + async def _plan_and_act(self) -> Message: + """first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... Use llm to come up with the plan dynamically.""" + # TODO: to be implemented + return Message("") + + async def react(self) -> Message: + """Entry to one of three strategies by which Role reacts to the observed Message""" + if self._rc.react_mode == RoleReactMode.REACT: + rsp = await self._react() + elif self._rc.react_mode == RoleReactMode.BY_ORDER: + rsp = await self._act_by_order() + elif self._rc.react_mode == RoleReactMode.PLAN_AND_ACT: + rsp = await self._plan_and_act() + self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None + return rsp def recv(self, message: Message) -> None: """add message to history.""" @@ -223,6 +303,10 @@ class Role: return await self._react() + def get_memories(self, k=0) -> list[Message]: + """A wrapper to return the most recent k memories of this role, return all when k=0""" + return self._rc.memory.get(k=k) + async def run(self, message=None): """Observe, and think and act based on the results of the observation""" if message: @@ -237,7 +321,7 @@ class Role: logger.debug(f"{self._setting}: no news. waiting.") return - rsp = await self._react() + rsp = await self.react() # Publish the reply to the environment, waiting for the next subscriber to process self._publish_message(rsp) return rsp diff --git a/metagpt/software_company.py b/metagpt/software_company.py index b2bd18c58..d44a0068a 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -5,58 +5,9 @@ @Author : alexanderwu @File : software_company.py """ -from pydantic import BaseModel, Field +from metagpt.team import Team as SoftwareCompany -from metagpt.actions import BossRequirement -from metagpt.config import CONFIG -from metagpt.environment import Environment -from metagpt.logs import logger -from metagpt.roles import Role -from metagpt.schema import Message -from metagpt.utils.common import NoMoneyException - - -class SoftwareCompany(BaseModel): - """ - Software Company: Possesses a team, SOP (Standard Operating Procedures), and a platform for instant messaging, - dedicated to writing executable code. - """ - environment: Environment = Field(default_factory=Environment) - investment: float = Field(default=10.0) - idea: str = Field(default="") - - class Config: - arbitrary_types_allowed = True - - def hire(self, roles: list[Role]): - """Hire roles to cooperate""" - self.environment.add_roles(roles) - - def invest(self, investment: float): - """Invest company. raise NoMoneyException when exceed max_budget.""" - self.investment = investment - CONFIG.max_budget = investment - logger.info(f'Investment: ${investment}.') - - def _check_balance(self): - if CONFIG.total_cost > CONFIG.max_budget: - raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - - def start_project(self, idea): - """Start a project from publishing boss requirement.""" - self.idea = idea - self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) - - def _save(self): - logger.info(self.json()) - - async def run(self, n_round=3): - """Run company until target round or no money""" - while n_round > 0: - # self._save() - n_round -= 1 - logger.debug(f"{n_round=}") - self._check_balance() - await self.environment.run() - return self.environment.history - \ No newline at end of file +import warnings +warnings.warn("metagpt.software_company is deprecated and will be removed in the future" + "Please use metagpt.team instead. SoftwareCompany class is now named as Team.", + DeprecationWarning, 2) diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 5e5b275b0..204c22c67 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -34,7 +34,10 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, engine = CONFIG.mermaid_engine.lower() if engine == "nodejs": if check_cmd_exists(CONFIG.mmdc) != 0: - logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") + logger.warning( + "RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc," + "or consider changing MERMAID_ENGINE to `playwright`, `pyppeteer`, or `ink`." + ) return -1 for suffix in ["pdf", "svg", "png"]: diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index a5a65803a..1af96f272 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -22,6 +22,7 @@ TOKEN_COSTS = { "gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12}, "gpt-4-0613": {"prompt": 0.06, "completion": 0.12}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, + "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069} # 32k version, prompt + completion tokens=0.005￥/k-tokens } @@ -37,6 +38,7 @@ TOKEN_MAX = { "gpt-4-32k-0314": 32768, "gpt-4-0613": 8192, "text-embedding-ada-002": 8192, + "chatglm_turbo": 32768 } @@ -68,7 +70,9 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): return count_message_tokens(messages, model="gpt-4-0613") else: raise NotImplementedError( - f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.""" + f"num_tokens_from_messages() is not implemented for model {model}. " + f"See https://github.com/openai/openai-python/blob/main/chatml.md " + f"for information on how messages are converted to tokens." ) num_tokens = 0 for message in messages: diff --git a/requirements.txt b/requirements.txt index 24a2d94c3..f0169d7fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ langchain==0.0.231 loguru==0.6.0 meilisearch==0.21.0 numpy==1.24.3 -openai +openai>=0.28.0 openpyxl beautifulsoup4==4.12.2 pandas==2.0.3 @@ -44,4 +44,4 @@ ta==0.10.2 semantic-kernel==0.3.13.dev0 wrapt==1.15.0 websocket-client==0.58.0 - +zhipuai==1.0.7 diff --git a/setup.py b/setup.py index f9ae768e6..239156ae3 100644 --- a/setup.py +++ b/setup.py @@ -30,16 +30,16 @@ with open(path.join(here, "requirements.txt"), encoding="utf-8") as f: setup( name="metagpt", - version="0.1", + version="0.3.0", description="The Multi-Role Meta Programming Framework", long_description=long_description, long_description_content_type="text/markdown", - url="https://gitlab.deepwisdomai.com/pub/metagpt", + url="https://github.com/geekan/MetaGPT", author="Alexander Wu", author_email="alexanderwu@fuzhi.ai", license="Apache 2.0", keywords="metagpt multi-role multi-agent programming gpt llm", - packages=find_packages(exclude=["contrib", "docs", "examples"]), + packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, extras_require={ diff --git a/startup.py b/startup.py index e2a903c9b..e9fbf94d3 100644 --- a/startup.py +++ b/startup.py @@ -11,7 +11,7 @@ from metagpt.roles import ( ProjectManager, QaEngineer, ) -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team async def startup( @@ -23,7 +23,7 @@ async def startup( implement: bool = True, ): """Run a startup. Be a boss.""" - company = SoftwareCompany() + company = Team() company.hire( [ ProductManager(), diff --git a/tests/metagpt/roles/test_ui.py b/tests/metagpt/roles/test_ui.py index 285bff323..d58d31bd9 100644 --- a/tests/metagpt/roles/test_ui.py +++ b/tests/metagpt/roles/test_ui.py @@ -2,7 +2,7 @@ # @Date : 2023/7/22 02:40 # @Author : stellahong (stellahong@fuzhi.ai) # -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team from metagpt.roles import ProductManager from tests.metagpt.roles.ui_role import UI @@ -15,7 +15,7 @@ def test_add_ui(): async def test_ui_role(idea: str, investment: float = 3.0, n_round: int = 5): """Run a startup. Be a boss.""" - company = SoftwareCompany() + company = Team() company.hire([ProductManager(), UI()]) company.invest(investment) company.start_project(idea) diff --git a/tests/metagpt/test_software_company.py b/tests/metagpt/test_software_company.py index 00538442c..4fc651f52 100644 --- a/tests/metagpt/test_software_company.py +++ b/tests/metagpt/test_software_company.py @@ -8,12 +8,12 @@ import pytest from metagpt.logs import logger -from metagpt.software_company import SoftwareCompany +from metagpt.team import Team @pytest.mark.asyncio -async def test_software_company(): - company = SoftwareCompany() +async def test_team(): + company = Team() company.start_project("做一个基础搜索引擎，可以支持知识库") history = await company.run(n_round=5) logger.info(history) From 45e48c8093e4bae519f62adc3b51e3c74ada1976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 17:44:16 +0800 Subject: [PATCH 107/135] feat: merge geekan:main --- .pre-commit-config.yam_ => .pre-commit-config.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .pre-commit-config.yam_ => .pre-commit-config.yaml (100%) diff --git a/.pre-commit-config.yam_ b/.pre-commit-config.yaml similarity index 100% rename from .pre-commit-config.yam_ rename to .pre-commit-config.yaml From ef9a925281e0a06ab910d14dcd5bc48a9689cc94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:11:12 +0800 Subject: [PATCH 108/135] feat: + gitpython 3.1.40 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 73a03d537..e72efc76c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -45,3 +45,4 @@ semantic-kernel==0.3.13.dev0 wrapt==1.15.0 websocket-client==0.58.0 aiofiles==23.2.1 +gitpython==3.1.40 \ No newline at end of file From 4c99107a333d6e9dc6bb52399de578364002ac4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:23:20 +0800 Subject: [PATCH 109/135] =?UTF-8?q?refactor:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/debug_error.py | 7 +++--- metagpt/utils/file_repository.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index e4a15d38d..dd1527154 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -16,6 +16,7 @@ from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger from metagpt.schema import RunCodeResult from metagpt.utils.common import CodeParser +from metagpt.utils.file_repository import FileRepository PROMPT_TEMPLATE = """ NOTICE @@ -50,7 +51,7 @@ class DebugError(Action): super().__init__(name, context, llm) async def run(self, *args, **kwargs) -> str: - output_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get(self.context.output_filename) + output_doc = await FileRepository.get_file(filename=self.context.output_filename, relative_path=TEST_OUTPUTS_FILE_REPO) if not output_doc: return "" output_detail = RunCodeResult.loads(output_doc.content) @@ -60,10 +61,10 @@ class DebugError(Action): return "" logger.info(f"Debug and rewrite {self.context.code_filename}") - code_doc = await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).get(self.context.code_filename) + code_doc = await FileRepository.get_file(filename=self.context.code_filename, relative_path=CONFIG.src_workspace) if not code_doc: return "" - test_doc = await CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get(self.context.test_filename) + test_doc = await FileRepository.get_file(filename=self.context.test_filename, relative_path=TEST_CODES_FILE_REPO) if not test_doc: return "" prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr) diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 8de4bdf5b..3df53cca3 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -16,6 +16,7 @@ from typing import Dict, List, Set import aiofiles +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Document from metagpt.utils.json_to_markdown import json_to_markdown @@ -186,3 +187,44 @@ class FileRepository: filename = Path(doc.filename).with_suffix(".md") await self.save(filename=str(filename), content=json_to_markdown(m)) logger.info(f"File Saved: {str(filename)}") + + @staticmethod + async def get_file(filename: Path | str, relative_path: Path | str = ".") -> Document | None: + """Retrieve a specific file from the file repository. + + :param filename: The name or path of the file to retrieve. + :type filename: Path or str + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + :return: The document representing the file, or None if not found. + :rtype: Document or None + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.get(filename=filename) + + @staticmethod + async def get_all_files(relative_path: Path | str = ".") -> List[Document]: + """Retrieve all files from the file repository. + + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + :return: A list of documents representing all files in the repository. + :rtype: List[Document] + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.get_all() + + @staticmethod + async def save_file(filename: Path | str, content, dependencies: List[str] = None, relative_path: Path | str = "."): + """Save a file to the file repository. + + :param filename: The name or path of the file to save. + :type filename: Path or str + :param content: The content of the file. + :param dependencies: A list of dependencies for the file. + :type dependencies: List[str], optional + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.save(filename=filename, content=content, dependencies=dependencies) From 81e719faa21bd08d3f97545c30d48658348afc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:29:29 +0800 Subject: [PATCH 110/135] =?UTF-8?q?refactor:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/prepare_documents.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index c9b60ff27..92d5730b2 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -24,8 +24,7 @@ class PrepareDocuments(Action): async def run(self, with_messages, **kwargs): if CONFIG.git_repo: - docs_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) - doc = await docs_repo.get(REQUIREMENT_FILENAME) + doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) # Create and initialize the workspace folder, initialize the Git environment. @@ -34,9 +33,8 @@ class PrepareDocuments(Action): CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. - docs_file_repository = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) - await docs_file_repository.save(REQUIREMENT_FILENAME, content=doc.content) + await FileRepository.save_file(filename=REQUIREMENT_FILENAME, content=doc.content, relative_path=DOCS_FILE_REPO) # Send a Message notification to the WritePRD action, instructing it to process requirements using # `docs/requirement.txt` and `docs/prds/`. From fa675ea3157a4a4a1b09209f52e4714d7a5e60d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:32:33 +0800 Subject: [PATCH 111/135] =?UTF-8?q?refactor:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/project_management.py | 9 ++++----- metagpt/const.py | 2 ++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 641d21533..d679a730c 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -15,7 +15,7 @@ from typing import List from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, TASK_PDF_FILE_REPO +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, TASK_PDF_FILE_REPO, PACKAGE_REQUIREMENTS_FILENAME from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.get_template import get_template @@ -263,16 +263,15 @@ class WriteTasks(Action): m = json.loads(doc.content) packages = set(m.get("Required Python third-party packages", set())) file_repo = CONFIG.git_repo.new_file_repository() - filename = "requirements.txt" - requirement_doc = await file_repo.get(filename) + requirement_doc = await file_repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME) if not requirement_doc: - requirement_doc = Document(filename=filename, root_path=".", content="") + requirement_doc = Document(filename=PACKAGE_REQUIREMENTS_FILENAME, root_path=".", content="") lines = requirement_doc.content.splitlines() for pkg in lines: if pkg == "": continue packages.add(pkg) - await file_repo.save(filename, content="\n".join(packages)) + await file_repo.save(PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages)) @staticmethod async def _save_pdf(task_doc): diff --git a/metagpt/const.py b/metagpt/const.py index a8c7356ca..ce06655f1 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -53,6 +53,8 @@ MESSAGE_ROUTE_TO_ALL = "" MESSAGE_ROUTE_TO_NONE = "" REQUIREMENT_FILENAME = "requirement.txt" +PACKAGE_REQUIREMENTS_FILENAME = "requirements.txt" + DOCS_FILE_REPO = "docs" PRDS_FILE_REPO = "docs/prds" SYSTEM_DESIGN_FILE_REPO = "docs/system_design" From 726eadf1cce7205fbbe960d30326c5eb118c09a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 19:54:41 +0800 Subject: [PATCH 112/135] =?UTF-8?q?refactor:=20=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/design_api.py | 4 +-- metagpt/actions/prepare_documents.py | 2 +- metagpt/actions/project_management.py | 4 +-- metagpt/actions/write_code.py | 6 ++--- metagpt/actions/write_prd.py | 8 +++--- metagpt/utils/file_repository.py | 39 +++++++++++++++++++++++---- 6 files changed, 45 insertions(+), 18 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 021edfe72..f987c6042 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -24,6 +24,7 @@ from metagpt.const import ( from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser +from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -304,8 +305,7 @@ class WriteDesign(Action): @staticmethod async def _save_pdf(design_doc): - file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) - await file_repo.save_pdf(doc=design_doc) + await FileRepository.save_as(doc=design_doc, with_suffix=".md", relative_path=SYSTEM_DESIGN_PDF_FILE_REPO) @staticmethod async def _save_mermaid_file(data: str, pathname: Path): diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 92d5730b2..30558c93f 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -29,7 +29,7 @@ class PrepareDocuments(Action): # Create and initialize the workspace folder, initialize the Git environment. CONFIG.git_repo = GitRepository() - workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_file_name() + workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_filename() CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index d679a730c..7205d11e7 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -18,6 +18,7 @@ from metagpt.config import CONFIG from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, TASK_PDF_FILE_REPO, PACKAGE_REQUIREMENTS_FILENAME from metagpt.logs import logger from metagpt.schema import Document, Documents +from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template templates = { @@ -275,8 +276,7 @@ class WriteTasks(Action): @staticmethod async def _save_pdf(task_doc): - file_repo = CONFIG.git_repo.new_file_repository(TASK_PDF_FILE_REPO) - await file_repo.save_pdf(doc=task_doc) + await FileRepository.save_as(doc=task_doc, with_suffix=".md", relative_path=TASK_PDF_FILE_REPO) class AssignTasks(Action): diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index e9d41bb20..3a4ca7768 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -23,6 +23,7 @@ from metagpt.const import TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger from metagpt.schema import CodingContext, RunCodeResult from metagpt.utils.common import CodeParser +from metagpt.utils.file_repository import FileRepository PROMPT_TEMPLATE = """ NOTICE @@ -82,9 +83,8 @@ class WriteCode(Action): async def run(self, *args, **kwargs) -> CodingContext: coding_context = CodingContext.loads(self.context.content) - test_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get( - "test_" + coding_context.filename + ".json" - ) + test_doc = await FileRepository.get_file(filename="test_" + coding_context.filename + ".json", + relative_path=TEST_OUTPUTS_FILE_REPO) logs = "" if test_doc: test_detail = RunCodeResult.loads(test_doc.content) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index cc21058b4..c1653a850 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -290,8 +290,7 @@ class WritePRD(Action): async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are # related to the PRD. If they are related, rewrite the PRD. - docs_file_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) - requirement_doc = await docs_file_repo.get(REQUIREMENT_FILENAME) + requirement_doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) prd_docs = await prds_file_repo.get_all() change_files = Documents() @@ -355,7 +354,7 @@ class WritePRD(Action): prd = await self._run_new_requirement(requirements=[requirement_doc.content], *args, **kwargs) new_prd_doc = Document( root_path=PRDS_FILE_REPO, - filename=FileRepository.new_file_name() + ".json", + filename=FileRepository.new_filename() + ".json", content=prd.instruct_content.json(), ) elif await self._is_relative_to(requirement_doc, prd_doc): @@ -382,5 +381,4 @@ class WritePRD(Action): @staticmethod async def _save_pdf(prd_doc): - file_repo = CONFIG.git_repo.new_file_repository(PRD_PDF_FILE_REPO) - await file_repo.save_pdf(doc=prd_doc) + await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 3df53cca3..018cac168 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -168,7 +168,7 @@ class FileRepository: return children @staticmethod - def new_file_name(): + def new_filename(): """Generate a new filename based on the current timestamp and a UUID suffix. :return: A new filename string. @@ -178,14 +178,22 @@ class FileRepository: # guid_suffix = str(uuid.uuid4())[:8] # return f"{current_time}x{guid_suffix}" - async def save_pdf(self, doc: Document): - """Save a Document as a PDF file. + async def save_doc(self, doc: Document, with_suffix:str = None, dependencies: List[str] = None): + """Save a Document instance as a PDF file. + + This method converts the content of the Document instance to Markdown, + saves it to a file with an optional specified suffix, and logs the saved file. :param doc: The Document instance to be saved. + :type doc: Document + :param with_suffix: An optional suffix to append to the saved file's name. + :type with_suffix: str, optional + :param dependencies: A list of dependencies for the saved file. + :type dependencies: List[str], optional """ m = json.loads(doc.content) - filename = Path(doc.filename).with_suffix(".md") - await self.save(filename=str(filename), content=json_to_markdown(m)) + filename = Path(doc.filename).with_suffix(with_suffix) if with_suffix is not None else Path(doc.filename) + await self.save(filename=str(filename), content=json_to_markdown(m), dependencies=dependencies) logger.info(f"File Saved: {str(filename)}") @staticmethod @@ -228,3 +236,24 @@ class FileRepository: """ file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) return await file_repo.save(filename=filename, content=content, dependencies=dependencies) + + @staticmethod + async def save_as(doc:Document, with_suffix:str = None, dependencies: List[str] = None, relative_path: Path | str = "."): + """Save a Document instance with optional modifications. + + This static method creates a new FileRepository, saves the Document instance + with optional modifications (such as a suffix), and logs the saved file. + + :param doc: The Document instance to be saved. + :type doc: Document + :param with_suffix: An optional suffix to append to the saved file's name. + :type with_suffix: str, optional + :param dependencies: A list of dependencies for the saved file. + :type dependencies: List[str], optional + :param relative_path: The relative path within the file repository. + :type relative_path: Path or str, optional + :return: A boolean indicating whether the save operation was successful. + :rtype: bool + """ + file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path) + return await file_repo.save_doc(doc=doc, with_suffix=with_suffix, dependencies=dependencies) From 2cd7d266ddce6a4c0979e29363d38b6c58f9b15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 27 Nov 2023 21:20:46 +0800 Subject: [PATCH 113/135] feat: merge Config class of send18:dev branch --- metagpt/actions/run_code.py | 6 +++-- metagpt/config.py | 52 ++++++++++++++++++++++++++++++------- metagpt/const.py | 3 +++ 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 1e7010e52..fa13a0980 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -12,13 +12,15 @@ RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError. 4. According to section 2.2.3.5.7 of RFC 135, change the method of transferring file content (code files, unit test files, log files) from using the message to using the file name. + 5. Merged the `Config` class of send18:dev branch to take over the set/get operations of the Environment + class. """ -import os import subprocess import traceback from typing import Tuple from metagpt.actions.action import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import RunCodeResult @@ -92,7 +94,7 @@ class RunCode(Action): additional_python_paths = [str(path) for path in additional_python_paths] # Copy the current environment variables - env = os.environ.copy() + env = CONFIG.new_environ() # Modify the PYTHONPATH environment variable additional_python_paths = [working_directory] + additional_python_paths diff --git a/metagpt/config.py b/metagpt/config.py index a20f58ec1..1b70d5fa6 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -7,11 +7,13 @@ Provide configuration, singleton 2. Add the parameter `src_workspace` for the old version project path. """ import os +from copy import deepcopy +from typing import Any import openai import yaml -from metagpt.const import PROJECT_ROOT +from metagpt.const import OPTIONS, PROJECT_ROOT from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.singleton import Singleton @@ -42,9 +44,11 @@ class Config(metaclass=Singleton): default_yaml_file = PROJECT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): - self._configs = {} - self._init_with_config_files_and_env(self._configs, yaml_file) + self._init_with_config_files_and_env(yaml_file) logger.info("Config loading done.") + self._update() + + def _update(self): self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") @@ -96,12 +100,10 @@ class Config(metaclass=Singleton): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") - self.git_repo = None - self.src_workspace = None - def _init_with_config_files_and_env(self, configs: dict, yaml_file): + def _init_with_config_files_and_env(self, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" - configs.update(os.environ) + configs = dict(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: if not _yaml_file.exists(): @@ -112,11 +114,13 @@ class Config(metaclass=Singleton): yaml_data = yaml.safe_load(file) if not yaml_data: continue - os.environ.update({k: v for k, v in yaml_data.items() if isinstance(v, str)}) configs.update(yaml_data) + OPTIONS.set(configs) - def _get(self, *args, **kwargs): - return self._configs.get(*args, **kwargs) + @staticmethod + def _get(*args, **kwargs): + m = OPTIONS.get() + return m.get(*args, **kwargs) def get(self, key, *args, **kwargs): """Search for a value in config/key.yaml, config/config.yaml, and env; raise an error if not found""" @@ -125,5 +129,33 @@ class Config(metaclass=Singleton): raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value + def __setattr__(self, name: str, value: Any) -> None: + OPTIONS.get()[name] = value + + def __getattr__(self, name: str) -> Any: + m = OPTIONS.get() + return m.get(name) + + def set_context(self, options: dict): + """Update current config""" + if not options: + return + opts = deepcopy(OPTIONS.get()) + opts.update(options) + OPTIONS.set(opts) + self._update() + + @property + def options(self): + """Return all key-values""" + return OPTIONS.get() + + def new_environ(self): + """Return a new os.environ object""" + env = os.environ.copy() + m = self.options + env.update({k: v for k, v in m.items() if isinstance(v, str)}) + return env + CONFIG = Config() diff --git a/metagpt/const.py b/metagpt/const.py index ce06655f1..9278a5d0e 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -8,6 +8,7 @@ common properties in the Message. @Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. """ +import contextvars from pathlib import Path @@ -27,6 +28,8 @@ def get_project_root(): current_path = parent_path +OPTIONS = contextvars.ContextVar("OPTIONS") + PROJECT_ROOT = get_project_root() DATA_PATH = PROJECT_ROOT / "data" WORKSPACE_ROOT = PROJECT_ROOT / "workspace" From 49f0b5e9f140c5d1a9a1b88289a0488a465601b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 28 Nov 2023 16:41:32 +0800 Subject: [PATCH 114/135] feat: fix memory.add --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index f7de58d5a..1c9da7e6c 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -214,7 +214,7 @@ class Role: ) else: msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) - self._rc.memory.add(msg) + self._rc.memory.add(msg) return msg From db9e900838b2c8eac4558fc858ce4a4e0cee6e62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 09:52:26 +0800 Subject: [PATCH 115/135] feat: merge geekan:cli-etc --- metagpt/llm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/llm.py b/metagpt/llm.py index 14bbad1b4..a35ba354b 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -8,10 +8,13 @@ from metagpt.config import CONFIG from metagpt.provider.anthropic_api import Claude2 as Claude +from metagpt.provider.human_provider import HumanProvider from metagpt.provider.openai_api import OpenAIGPTAPI from metagpt.provider.spark_api import SparkAPI from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +_ = HumanProvider() + def LLM() -> "BaseGPTAPI": """initialize different LLM instance according to the key field existence""" From f564bb540a97cc76229a46d51d0cde21980770ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 09:52:55 +0800 Subject: [PATCH 116/135] feat: merge geekan:cli-etc --- metagpt/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index a35ba354b..d8d06c0a1 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -13,7 +13,7 @@ from metagpt.provider.openai_api import OpenAIGPTAPI from metagpt.provider.spark_api import SparkAPI from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI -_ = HumanProvider() +_ = HumanProvider() # Avoid pre-commit error def LLM() -> "BaseGPTAPI": From eff1cb7dc1ef842fc55d9118386052eadb98cf93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 10:14:04 +0800 Subject: [PATCH 117/135] feat: Add 'id' to 'Message' according to Section 2.2.3.1.1 of RFC 135. --- metagpt/schema.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 9e5854997..d1174799a 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -10,12 +10,14 @@ 1. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. 2. Encapsulate the common key-values set to pydantic structures to standardize and unify parameter passing between actions. + 3. Add `id` to `Message` according to Section 2.2.3.1.1 of RFC 135. """ from __future__ import annotations import asyncio import json import os.path +import uuid from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError from pathlib import Path @@ -86,6 +88,7 @@ class Documents(BaseModel): class Message(BaseModel): """list[: ]""" + id: str # According to Section 2.2.3.1.1 of RFC 135 content: str instruct_content: BaseModel = Field(default=None) role: str = "user" # system / user / assistant @@ -113,6 +116,7 @@ class Message(BaseModel): :param role: Message meta info tells who sent this message. """ super().__init__( + id=uuid.uuid4().hex, content=content, instruct_content=instruct_content, role=role, From 7b44fccf8d826aa881b2fdd1765343a9a7207c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 16:22:05 +0800 Subject: [PATCH 118/135] feat: merge geekan:cli-etc --- metagpt/actions/prepare_documents.py | 7 ++- metagpt/startup.py | 16 +++++- metagpt/team.py | 11 ++-- metagpt/utils/git_repository.py | 4 +- startup.py | 80 ---------------------------- 5 files changed, 26 insertions(+), 92 deletions(-) delete mode 100644 startup.py diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 30558c93f..fe954b79c 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -12,7 +12,7 @@ from pathlib import Path from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME, WORKSPACE_ROOT +from metagpt.const import DEFAULT_WORKSPACE_ROOT, DOCS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.schema import Document from metagpt.utils.file_repository import FileRepository from metagpt.utils.git_repository import GitRepository @@ -28,8 +28,11 @@ class PrepareDocuments(Action): return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) # Create and initialize the workspace folder, initialize the Git environment. + default_workspace_root = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT + default_project_name = CONFIG.project_name or FileRepository.new_filename() + default_workdir = Path(default_workspace_root) / default_project_name CONFIG.git_repo = GitRepository() - workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_filename() + workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else default_workdir CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. diff --git a/metagpt/startup.py b/metagpt/startup.py index 35b9b8b66..de348780b 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -4,6 +4,8 @@ import asyncio import typer +from metagpt.config import CONFIG + app = typer.Typer() @@ -17,6 +19,10 @@ def startup( implement: bool = typer.Option(True, help="Enable or disable code implementation."), project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'."), inc: bool = typer.Option(False, help="Incremental mode. Use it to coop with existing repo."), + project_path: str = typer.Option( + help="Specify the directory path of the old version project to fulfill the " "incremental requirements." + ), + reqa_file: str = typer.Option(help="Specify the source file name for rewriting the quality test code."), ): """Run a startup. Be a boss.""" from metagpt.roles import ( @@ -28,6 +34,12 @@ def startup( ) from metagpt.team import Team + # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135. + CONFIG.project_name = project_name + CONFIG.inc = inc + CONFIG.project_path = project_path + CONFIG.reqa_file = reqa_file + company = Team() company.hire( [ @@ -44,9 +56,9 @@ def startup( company.hire([QaEngineer()]) company.invest(investment) - company.run_project(idea, project_name=project_name, inc=inc) + company.run_project(idea) asyncio.run(company.run(n_round=n_round)) if __name__ == "__main__": - startup(idea="Make a 2048 game.") + app() diff --git a/metagpt/team.py b/metagpt/team.py index e252935c4..92f379c97 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -11,6 +11,7 @@ from pydantic import BaseModel, Field from metagpt.actions import UserRequirement from metagpt.config import CONFIG +from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Role @@ -45,16 +46,14 @@ class Team(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f"Insufficient funds: {CONFIG.max_budget}") - def run_project(self, idea, send_to: str = "", project_name: str = "", inc: bool = False): + def run_project(self, idea, send_to: str = ""): """Start a project from publishing user requirement.""" self.idea = idea - # If user set project_name, then use it. - if project_name: - path = CONFIG.workspace_path / project_name - self.env.load_existing_repo(path, inc=inc) # Human requirement. - self.env.publish_message(Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to)) + self.env.publish_message( + Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL) + ) def _save(self): logger.info(self.json(ensure_ascii=False)) diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index b8e35199b..b1cfe1ed2 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -16,7 +16,7 @@ from typing import Dict from git.repo import Repo from git.repo.fun import is_git_dir -from metagpt.const import WORKSPACE_ROOT +from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.dependency_file import DependencyFile from metagpt.utils.file_repository import FileRepository @@ -201,7 +201,7 @@ class GitRepository: if __name__ == "__main__": - path = WORKSPACE_ROOT / "git" + path = DEFAULT_WORKSPACE_ROOT / "git" path.mkdir(exist_ok=True, parents=True) repo = GitRepository() diff --git a/startup.py b/startup.py deleted file mode 100644 index 1a59e7fa2..000000000 --- a/startup.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import asyncio - -import fire - -from metagpt.config import CONFIG -from metagpt.roles import ( - Architect, - Engineer, - ProductManager, - ProjectManager, - QaEngineer, -) -from metagpt.software_company import SoftwareCompany - - -async def startup( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = False, - run_tests: bool = False, - implement: bool = True, -): - """Run a startup. Be a boss.""" - company = SoftwareCompany() - company.hire( - [ - ProductManager(), - Architect(), - ProjectManager(), - ] - ) - - # if implement or code_review - if implement or code_review: - # developing features: implement the idea - company.hire([Engineer(n_borg=5, use_code_review=code_review)]) - - if run_tests: - # developing features: run tests on the spot and identify bugs - # (bug fixing capability comes soon!) - company.hire([QaEngineer()]) - - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) - - -def main( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = True, - run_tests: bool = False, - implement: bool = True, - project_path: str = None, - reqa_file: str = None, -): - """ - We are a software startup comprised of AI. By investing in us, - you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea, such as "Creating a snake game." - :param investment: As an investor, you have the opportunity to contribute - a certain dollar amount to this AI company. - :param n_round: - :param code_review: Whether to use code review. - :param run_tests: Whether run unit tests. - :param implement: Whether to write codes. - :param project_path: The path of the old version project to improve. - :return: - """ - CONFIG.WORKDIR = project_path - CONFIG.REQA_FILENAME = reqa_file - asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) - - -if __name__ == "__main__": - fire.Fire(main) From 94043a89f41fa5da81d1fc56e0a1866423ae87d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 20:12:03 +0800 Subject: [PATCH 119/135] feat: merge geekan:cli-etc --- metagpt/actions/prepare_documents.py | 8 ++------ metagpt/startup.py | 19 ++++++++++--------- metagpt/utils/file_repository.py | 4 ++-- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index fe954b79c..71c94d25a 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -8,8 +8,6 @@ RFC 135 2.2.3.5.1. """ -from pathlib import Path - from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG from metagpt.const import DEFAULT_WORKSPACE_ROOT, DOCS_FILE_REPO, REQUIREMENT_FILENAME @@ -28,11 +26,9 @@ class PrepareDocuments(Action): return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) # Create and initialize the workspace folder, initialize the Git environment. - default_workspace_root = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT - default_project_name = CONFIG.project_name or FileRepository.new_filename() - default_workdir = Path(default_workspace_root) / default_project_name + project_name = CONFIG.project_name or FileRepository.new_filename() + workdir = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name CONFIG.git_repo = GitRepository() - workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else default_workdir CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. diff --git a/metagpt/startup.py b/metagpt/startup.py index de348780b..78f32d556 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -12,17 +12,18 @@ app = typer.Typer() @app.command() def startup( idea: str = typer.Argument(..., help="Your innovative idea, such as 'Create a 2048 game.'"), - investment: float = typer.Option(3.0, help="Dollar amount to invest in the AI company."), - n_round: int = typer.Option(5, help="Number of rounds for the simulation."), - code_review: bool = typer.Option(True, help="Whether to use code review."), - run_tests: bool = typer.Option(False, help="Whether to enable QA for adding & running tests."), - implement: bool = typer.Option(True, help="Enable or disable code implementation."), - project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'."), - inc: bool = typer.Option(False, help="Incremental mode. Use it to coop with existing repo."), + investment: float = typer.Option(default=3.0, help="Dollar amount to invest in the AI company."), + n_round: int = typer.Option(default=5, help="Number of rounds for the simulation."), + code_review: bool = typer.Option(default=True, help="Whether to use code review."), + run_tests: bool = typer.Option(default=False, help="Whether to enable QA for adding & running tests."), + implement: bool = typer.Option(default=True, help="Enable or disable code implementation."), + project_name: str = typer.Option(default="", help="Unique project name, such as 'game_2048'."), + inc: bool = typer.Option(default=False, help="Incremental mode. Use it to coop with existing repo."), project_path: str = typer.Option( - help="Specify the directory path of the old version project to fulfill the " "incremental requirements." + default="", + help="Specify the directory path of the old version project to fulfill the " "incremental requirements.", ), - reqa_file: str = typer.Option(help="Specify the source file name for rewriting the quality test code."), + reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), ): """Run a startup. Be a boss.""" from metagpt.roles import ( diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 846bfcd0c..0815bf90a 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -54,7 +54,7 @@ class FileRepository: """ pathname = self.workdir / filename pathname.parent.mkdir(parents=True, exist_ok=True) - async with aiofiles.open(str(pathname), mode="wb") as writer: + async with aiofiles.open(str(pathname), mode="w") as writer: await writer.write(content) logger.info(f"save to: {str(pathname)}") @@ -98,7 +98,7 @@ class FileRepository: if not path_name.exists(): return None try: - async with aiofiles.open(str(path_name), mode="rb") as reader: + async with aiofiles.open(str(path_name), mode="r") as reader: doc.content = await reader.read() except FileNotFoundError as e: logger.info(f"open {str(path_name)} failed:{e}") From 09b6d2df8377883335696c50f21d8956518f75ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 20:26:26 +0800 Subject: [PATCH 120/135] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 431879c25..2b9c20047 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -281,11 +281,7 @@ class WriteDesign(Action): # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" # contain space, have to use setattr - setattr( - system_design.instruct_content, - "project_name", - system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - ) + self._rename_project_name(system_design=system_design) await self._rename_workspace(system_design) # ======= # # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" contain space, have to use setattr @@ -303,17 +299,29 @@ class WriteDesign(Action): system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python # package name" contain space, have to use setattr + self._rename_project_name(system_design=system_design) + system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) + return system_design_doc + + @staticmethod + def _rename_project_name(system_design): + if CONFIG.project_name: + setattr( + system_design.instruct_content, + "project_name", + CONFIG.project_name, + ) + return setattr( system_design.instruct_content, "project_name", system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), ) - system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) - return system_design_doc @staticmethod async def _rename_workspace(system_design): - if CONFIG.WORKDIR: # Updating on the old version has already been specified if it's valid. + if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to + # Section 2.2.3.10 of RFC 135 return if isinstance(system_design, ActionOutput): From 6fa3deef00ad06c68c77d68e60b349e0d0137ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 20:58:41 +0800 Subject: [PATCH 121/135] feat: merge geekan:cli-etc --- metagpt/actions/write_code.py | 28 ++++++++++++---------------- metagpt/utils/git_repository.py | 1 + 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index e373b1127..0cd41c52f 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -31,22 +31,6 @@ Role: You are a professional engineer; the main goal is to write PEP8 compliant, Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". ------ -# Context -{context} ------ - -## Code: {filename} Write code with triple quoto, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. -5. Think before writing: What should be implemented and provided in this document? -6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. -7. Do not use public member functions that do not exist in your design. -8. Before using a variable, make sure you reference it first -9. Write out EVERY DETAIL, DON'T LEAVE TODO. - ----- # Design ```json @@ -68,6 +52,18 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc {logs} ``` ----- + +## Code: {filename} Write code with triple quoto, based on the following list and context. +1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. +2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets +3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. +5. Think before writing: What should be implemented and provided in this document? +6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +7. Do not use public member functions that do not exist in your design. +8. Before using a variable, make sure you reference it first +9. Write out EVERY DETAIL, DON'T LEAVE TODO. + ## Format example ----- ## Code: {filename} diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index b1cfe1ed2..7c9ec645f 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -59,6 +59,7 @@ class GitRepository: :param local_path: The local path to the Git repository. :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. """ + local_path = Path(local_path) if self.is_git_dir(local_path): self._repository = Repo(local_path) return From 810768a3505bb06dc9ae76024073432c671269ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 21:34:29 +0800 Subject: [PATCH 122/135] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 2b9c20047..3e17239b0 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -172,7 +172,7 @@ Max Output: 8192 chars or 2048 tokens. Try to use them up. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Python str with python triple quote, concise and clear, characters only use a combination of all lowercase and underscores ## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here From cd24931b65b4a6ca9e926174c0a86dbdc3b1856c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 22:06:40 +0800 Subject: [PATCH 123/135] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 3e17239b0..e31ea76a8 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -166,24 +166,22 @@ MERGE_PROMPT = """ {context} ----- -Role: You are an architect; The goal is to incrementally update the "Old Design" based on the information provided by the "Context," aiming to design a state-of-the-art (SOTA) Python system compliant with PEP8. Additionally, the objective is to optimize the use of high-quality open-source tools. -Requirement: Fill in the following missing information based on the context, each section name is a key in json -Max Output: 8192 chars or 2048 tokens. Try to use them up. +Role: You are an architect; The goal is to incrementally update the "Old Design" based on the information provided by the "Context," aiming to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. +Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately +ATTENTION: Output carefully referenced "Old Design" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quote, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Design" format, -and only output the json inside this tag, nothing else +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """ From e34b8bbf0bbb9d46757da1eb33883fd8b554df56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 22:13:10 +0800 Subject: [PATCH 124/135] feat: merge geekan:cli-etc --- metagpt/actions/write_prd.py | 47 ++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index ab216b7a0..c61684918 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -261,24 +261,41 @@ MERGE_PROMPT = """ {old_prd} ----- Role: You are a professional product manager; The goal is to merge the newly added requirements into the existing PRD in order to design a concise, usable, and efficient product. +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design +ATTENTION: Output carefully referenced "Old PRD" in format. -## Original Requirements: Provide as Plain text, place the polished complete original requirements here +## YOU NEED TO FULFILL THE BELOW JSON DOC -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less - -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower - -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +{{ + "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. + "Original Requirements": "", # str, place the polished complete original requirements here + "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Search Information": "", + "Requirements": "", + "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. + "User Stories": [], # Provided as Python list[str], up to 5 scenario-based user stories + "Competitive Analysis": [], # Provided as Python list[str], up to 8 competitive product analyses + # Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + "Competitive Quadrant Chart": "quadrantChart + title Reach and engagement of campaigns + x-axis Low Reach --> High Reach + y-axis Low Engagement --> High Engagement + quadrant-1 We should expand + quadrant-2 Need to promote + quadrant-3 Re-evaluate + quadrant-4 May be improved + Campaign A: [0.3, 0.6] + Campaign B: [0.45, 0.23] + Campaign C: [0.57, 0.69] + Campaign D: [0.78, 0.34] + Campaign E: [0.40, 0.34] + Campaign F: [0.35, 0.78]", + "Requirement Analysis": "", # Provide as Plain text. + "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], # Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards + "UI Design draft": "", # Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. + "Anything UNCLEAR": "", # Provide as Plain text. Try to clarify it. +}} output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old PRD" format, and only output the json inside this tag, nothing else From 6e0fc042258c005a8842cff66923d6bb76aa0e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 29 Nov 2023 22:25:47 +0800 Subject: [PATCH 125/135] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index e31ea76a8..7164cef26 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -182,6 +182,9 @@ ATTENTION: Output carefully referenced "Old Design" in format. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. ## Anything UNCLEAR: Provide as Plain text. Try to clarify it. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Design" format, +and only output the json inside this tag, nothing else """ From 4928a896ca8b357bdfbc6d08d3b72d86f4598995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 10:16:34 +0800 Subject: [PATCH 126/135] feat: merge geekan:cli-etc --- metagpt/actions/prepare_documents.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 71c94d25a..b339d897d 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -7,6 +7,7 @@ @Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. RFC 135 2.2.3.5.1. """ +import shutil from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG @@ -28,6 +29,8 @@ class PrepareDocuments(Action): # Create and initialize the workspace folder, initialize the Git environment. project_name = CONFIG.project_name or FileRepository.new_filename() workdir = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name + if not CONFIG.inc and workdir.exists(): + shutil.rmtree(workdir) CONFIG.git_repo = GitRepository() CONFIG.git_repo.open(local_path=workdir, auto_init=True) From 6010ce70f651d514db06f0014f07d12ec2e7c354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 12:13:45 +0800 Subject: [PATCH 127/135] feat: merge geekan:cli-etc --- metagpt/actions/prepare_documents.py | 3 ++- metagpt/actions/write_prd.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index b339d897d..3d202e762 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -8,6 +8,7 @@ RFC 135 2.2.3.5.1. """ import shutil +from pathlib import Path from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG @@ -28,7 +29,7 @@ class PrepareDocuments(Action): # Create and initialize the workspace folder, initialize the Git environment. project_name = CONFIG.project_name or FileRepository.new_filename() - workdir = CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name + workdir = Path(CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name) if not CONFIG.inc and workdir.exists(): shutil.rmtree(workdir) CONFIG.git_repo = GitRepository() diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index c61684918..3967a0578 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -260,7 +260,7 @@ MERGE_PROMPT = """ ## Old PRD {old_prd} ----- -Role: You are a professional product manager; The goal is to merge the newly added requirements into the existing PRD in order to design a concise, usable, and efficient product. +Role: You are a professional product manager; The goal is to incorporate the newly added requirements from the "Original Requirements" into the existing Product Requirements Document (PRD) in the "Old PRD" in order to design a concise, usable, and efficient product. Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design ATTENTION: Output carefully referenced "Old PRD" in format. From 5351b50d1cf963eebe6473783eb1860bdf6266c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 12:47:48 +0800 Subject: [PATCH 128/135] feat: merge geekan:cli-etc --- metagpt/actions/design_api.py | 62 ++--------------------------------- 1 file changed, 2 insertions(+), 60 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 7164cef26..c5787ba20 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -227,72 +227,12 @@ class WriteDesign(Action): # leaving room for global optimization in subsequent steps. return ActionOutput(content=changed_files.json(), instruct_content=changed_files) - # ======= - # def recreate_workspace(self, workspace: Path): - # try: - # shutil.rmtree(workspace) - # except FileNotFoundError: - # pass # Folder does not exist, but we don't care - # workspace.mkdir(parents=True, exist_ok=True) - - # async def _save_prd(self, docs_path, resources_path, context): - # prd_file = docs_path / "prd.md" - # if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: - # quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"] - # await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") - # - # if context[-1].instruct_content: - # logger.info(f"Saving PRD to {prd_file}") - # prd_file.write_text(context[-1].instruct_content.json(ensure_ascii=False), encoding='utf-8') - - # async def _save_system_design(self, docs_path, resources_path, system_design): - # data_api_design = system_design.instruct_content.dict()[ - # "Data structures and interfaces" - # ] # CodeParser.parse_code(block="Data structures and interfaces", text=content) - # seq_flow = system_design.instruct_content.dict()[ - # "Program call flow" - # ] # CodeParser.parse_code(block="Program call flow", text=content) - # await mermaid_to_file(data_api_design, resources_path / "data_api_design") - # await mermaid_to_file(seq_flow, resources_path / "seq_flow") - # system_design_file = docs_path / "system_design.md" - # logger.info(f"Saving System Designs to {system_design_file}") - # system_design_file.write_text(system_design.instruct_content.json(ensure_ascii=False), encoding='utf-8') - - # async def _save(self, context, system_design): - # if isinstance(system_design, ActionOutput): - # project_name = system_design.instruct_content.dict()["project_name"] - # else: - # project_name = CodeParser.parse_str(block="project_name", text=system_design) - # workspace = CONFIG.workspace_path / project_name - # self.recreate_workspace(workspace) - # docs_path = workspace / "docs" - # resources_path = workspace / "resources" - # docs_path.mkdir(parents=True, exist_ok=True) - # resources_path.mkdir(parents=True, exist_ok=True) - # await self._save_prd(docs_path, resources_path, context) - # await self._save_system_design(docs_path, resources_path, system_design) - - # async def run(self, context, format=CONFIG.prompt_format): - async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) - # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - - # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" - # contain space, have to use setattr self._rename_project_name(system_design=system_design) await self._rename_workspace(system_design) - # ======= - # # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" contain space, have to use setattr - # # setattr( - # # system_design.instruct_content, - # # "project_name", - # # system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - # # ) - # await self._save(context, system_design) - # >>>>>>> feature/geekan_cli_etc return system_design async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): @@ -306,6 +246,8 @@ class WriteDesign(Action): @staticmethod def _rename_project_name(system_design): + # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" + # contain space, have to use setattr if CONFIG.project_name: setattr( system_design.instruct_content, From 17bf646539ec2851c791c3b9cccabd8fdbf1753d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 12:53:02 +0800 Subject: [PATCH 129/135] feat: merge geekan:cli-etc --- metagpt/actions/project_management.py | 15 --------- metagpt/actions/write_code.py | 23 -------------- metagpt/actions/write_code_review.py | 17 ----------- metagpt/const.py | 28 ----------------- metagpt/environment.py | 44 --------------------------- metagpt/roles/engineer.py | 27 ---------------- metagpt/roles/qa_engineer.py | 26 ---------------- 7 files changed, 180 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index db856e55b..3d59daeed 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -258,21 +258,6 @@ class WriteTasks(Action): return task_doc async def _run_new_tasks(self, context, format=CONFIG.prompt_format): - # ======= - # def _save(self, context, rsp): - # if context[-1].instruct_content: - # ws_name = context[-1].instruct_content.dict()["project_name"] - # else: - # ws_name = CodeParser.parse_str(block="project_name", text=context[-1].content) - # file_path = CONFIG.workspace_path / ws_name / "docs/api_spec_and_tasks.md" - # file_path.write_text(rsp.instruct_content.json(ensure_ascii=False)) - # - # # Write requirements.txt - # requirements_path = CONFIG.workspace_path / ws_name / "requirements.txt" - # requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) - # - # async def run(self, context, format=CONFIG.prompt_format): - # >>>>>>> feature/geekan_cli_etc prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 0cd41c52f..59ccb49a5 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -79,29 +79,6 @@ class WriteCode(Action): def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) - # <<<<<<< HEAD - # ======= - # def _is_invalid(self, filename): - # return any(i in filename for i in ["mp3", "wav"]) - # - # def _save(self, context, filename, code): - # # logger.info(filename) - # # logger.info(code_rsp) - # if self._is_invalid(filename): - # return - # - # design = [i for i in context if i.cause_by == WriteDesign][0] - # - # ws_name = CodeParser.parse_str(block="project_name", text=design.content) - # ws_path = CONFIG.workspace_path / ws_name - # if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): - # ws_path = ws_path / ws_name - # code_path = ws_path / filename - # code_path.parent.mkdir(parents=True, exist_ok=True) - # code_path.write_text(code) - # logger.info(f"Saving Code to {code_path}") - # - # >>>>>>> feature/geekan_cli_etc @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 6d405029a..364f6af57 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -103,23 +103,6 @@ class WriteCodeReview(Action): code = CodeParser.parse_code(block="", text=code_rsp) return result, code - # <<<<<<< HEAD - # async def run(self, *args, **kwargs) -> CodingContext: - # format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) - # context = "\n----------\n".join( - # [self.context.design_doc.content, self.context.task_doc.content, self.context.code_doc.content] - # ) - # prompt = PROMPT_TEMPLATE.format( - # context=context, - # code=self.context.code_doc.content, - # filename=self.context.code_doc.filename, - # format_example=format_example, - # ) - # logger.info(f"Code review {self.context.code_doc.filename}..") - # code = await self.write_code(prompt) - # self.context.code_doc.content = code - # return self.context - # ======= async def run(self, *args, **kwargs) -> CodingContext: iterative_code = self.context.code_doc.content k = CONFIG.code_review_k_times or 1 diff --git a/metagpt/const.py b/metagpt/const.py index 6e616e820..a646cea7a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -19,34 +19,6 @@ import metagpt OPTIONS = contextvars.ContextVar("OPTIONS") -# <<<<<<< HEAD -# def get_project_root(): -# """Search upwards to find the project root directory.""" -# current_path = Path.cwd() -# while True: -# if ( -# (current_path / ".git").exists() -# or (current_path / ".project_root").exists() -# or (current_path / ".gitignore").exists() -# ): -# return current_path -# parent_path = current_path.parent -# if parent_path == current_path: -# raise Exception("Project root not found.") -# current_path = parent_path -# -# -# PROJECT_ROOT = get_project_root() -# DATA_PATH = PROJECT_ROOT / "data" -# WORKSPACE_ROOT = PROJECT_ROOT / "workspace" -# PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" -# UT_PATH = PROJECT_ROOT / "data/ut" -# SWAGGER_PATH = UT_PATH / "files/api/" -# UT_PY_PATH = UT_PATH / "files/ut/" -# API_QUESTIONS_PATH = UT_PATH / "files/question/" -# YAPI_URL = "http://yapi.deepwisdomai.com/" -# TMP = PROJECT_ROOT / "tmp" -# ======= def get_metagpt_package_root(): """Get the root directory of the installed package.""" package_root = Path(metagpt.__file__).parent.parent diff --git a/metagpt/environment.py b/metagpt/environment.py index e8bdd25c7..02eb3d340 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -23,7 +23,6 @@ from metagpt.utils.common import is_subscribed class Environment(BaseModel): - # <<<<<<< HEAD """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles @@ -32,17 +31,6 @@ class Environment(BaseModel): roles: dict[str, Role] = Field(default_factory=dict) members: dict[Role, Set] = Field(default_factory=dict) history: str = Field(default="") # For debug - # ======= - # """ - # Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - # """ - # - # roles: dict[str, Role] = Field(default_factory=dict) - # memory: Memory = Field(default_factory=Memory) # 已经私有化 - # history: str = Field(default='') - # repo: Repo = Field(default_factory=Repo) # 在CONFIG里 - # kv: dict = Field(default_factory=dict) # 在CONFIG里 - # >>>>>>> feature/geekan_cli_etc class Config: arbitrary_types_allowed = True @@ -83,38 +71,6 @@ class Environment(BaseModel): return True - # # Replaced by FileRepository.set_file - # def set_doc(self, content: str, filename: str): - # """向当前环境发布文档（包括代码）""" - # return self.repo.set(content, filename) - # - # # Replaced by FileRepository.get_file - # def get_doc(self, filename: str): - # return self.repo.get(filename) - # - # # Replaced by CONFIG.xx - # def set(self, k: str, v: str): - # self.kv[k] = v - # - # # Replaced by CONFIG.xx - # def get(self, k: str): - # return self.kv.get(k, None) - - # Replaced By 增量变更流程 - # def load_existing_repo(self, path: Path, inc: bool): - # self.repo = Repo.from_path(path) - # logger.info(self.repo.eda()) - # - # # Incremental mode: publish all docs to messages. Then roles can read the docs. - # if inc: - # docs = self.repo.get_text_documents() - # for doc in docs: - # msg = Message(content=doc.content) - # self.publish_message(msg) - # logger.info(f"Message from existing doc {doc.path}: {msg}") - # logger.info(f"Load {len(docs)} docs from existing repo.") - # raise NotImplementedError - async def run(self, k=1): """处理一次所有信息的运行 Process all Role runs at once diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 48262989c..78a7f3ba2 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -183,24 +183,6 @@ class Engineer(Role): msg = Message( content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode ) - # ======= - # context = [] - # msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - # for m in msg: - # context.append(m.content) - # context_str = "\n----------\n".join(context) - # # Write code - # code = await WriteCode().run(context=context_str, filename=todo) - # # Code review - # if self.use_code_review: - # # try: - # rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) - # code = rewrite_code - # # except Exception as e: - # # logger.error("code review failed!", e) - # file_path = self.write_file(todo, code) - # msg = Message(content=code, role=self.profile, cause_by=WriteCode) - # >>>>>>> feature/geekan_cli_etc self._rc.memory.add(msg) changed_files.add(coding_context.code_doc.filename) @@ -273,15 +255,6 @@ class Engineer(Role): coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json()) return coding_doc - # ======= - # async def _act(self) -> Message: - # """Determines the mode of action based on whether code review is used.""" - # logger.info(f"{self._setting}: ready to WriteCode") - # if self.use_code_review: - # return await self._act_sp_with_cr() - # return await self._act_sp() - # >>>>>>> feature/geekan_cli_etc - async def _new_code_actions(self): # Prepare file repos src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index fd2dcc786..ac5a280bb 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -44,32 +44,6 @@ class QaEngineer(Role): self.test_round = 0 self.test_round_allowed = test_round_allowed - # <<<<<<< HEAD - # ======= - # @classmethod - # def parse_workspace(cls, system_design_msg: Message) -> str: - # if system_design_msg.instruct_content: - # return system_design_msg.instruct_content.dict().get("project_name") - # return CodeParser.parse_str(block="project_name", text=system_design_msg.content) - # - # def get_workspace(self, return_proj_dir=True) -> Path: - # msg = self._rc.memory.get_by_action(WriteDesign)[-1] - # if not msg: - # return CONFIG.workspace_path / "src" - # workspace = self.parse_workspace(msg) - # # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. - # if return_proj_dir: - # return CONFIG.workspace_path / workspace - # # development codes directory: workspace/{package_name}/{package_name} - # return CONFIG.workspace_path / workspace / workspace - # - # def write_file(self, filename: str, code: str): - # workspace = self.get_workspace() / "tests" - # file = workspace / filename - # file.parent.mkdir(parents=True, exist_ok=True) - # file.write_text(code) - # - # >>>>>>> feature/geekan_cli_etc async def _write_test(self, message: Message) -> None: changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) From 6146d4dc7f352bc417bc2c720061613c7373ad39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 13:04:29 +0800 Subject: [PATCH 130/135] feat: merge RFC 135 --- metagpt/roles/engineer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 2bb1f3ea2..d42835a1b 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -218,15 +218,15 @@ class Engineer(Role): src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) changed_src_files = src_file_repo.changed_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). - summerizations = {} + summarizations = {} for filename in changed_src_files: - depenencies = src_file_repo.get_dependency(filename=filename) - ctx = CodeSummarizeContext.loads(filenames=depenencies) - if ctx not in summerizations: - summerizations[ctx] = set() - srcs = summerizations.get(ctx) + dependencies = src_file_repo.get_dependency(filename=filename) + ctx = CodeSummarizeContext.loads(filenames=dependencies) + if ctx not in summarizations: + summarizations[ctx] = set() + srcs = summarizations.get(ctx) srcs.add(filename) - for ctx, filenames in summerizations.items(): + for ctx, filenames in summarizations.items(): ctx.codes_filenames = filenames self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) if self.summarize_todos: From 6f3d1d6f5e6c6080bab47ad28184c698c5dd7913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 14:58:06 +0800 Subject: [PATCH 131/135] =?UTF-8?q?fixbug:=20=E5=A2=9E=E9=87=8F=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E6=97=B6=E9=9C=80=E6=B1=82=E6=B2=A1=E5=86=99=E5=85=A5?= =?UTF-8?q?docs/requirement.txt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/prepare_documents.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 30558c93f..8656de812 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -23,14 +23,11 @@ class PrepareDocuments(Action): super().__init__(name, context, llm) async def run(self, with_messages, **kwargs): - if CONFIG.git_repo: - doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) - return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) - - # Create and initialize the workspace folder, initialize the Git environment. - CONFIG.git_repo = GitRepository() - workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_filename() - CONFIG.git_repo.open(local_path=workdir, auto_init=True) + if not CONFIG.git_repo: + # Create and initialize the workspace folder, initialize the Git environment. + CONFIG.git_repo = GitRepository() + workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_filename() + CONFIG.git_repo.open(local_path=workdir, auto_init=True) # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) From 269eee4643728c5e5f0f6ce3efffa854f88c8f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 19:20:53 +0800 Subject: [PATCH 132/135] fixbug: The assumption that messages in 'memory' have been processed has been revoked. --- metagpt/roles/product_manager.py | 3 +++ metagpt/roles/role.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index bc6771829..966115c0f 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -50,3 +50,6 @@ class ProductManager(Role): else: self._set_state(0) return self._rc.todo + + async def _observe(self, ignore_memory=False) -> int: + return await super(ProductManager, self)._observe(ignore_memory=True) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1c9da7e6c..fe121ed1a 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -218,12 +218,12 @@ class Role: return msg - async def _observe(self) -> int: + async def _observe(self, ignore_memory=False) -> int: """Prepare new messages for processing from the message buffer and other sources.""" # Read unprocessed messages from the msg buffer. news = self._rc.msg_buffer.pop_all() # Store the read messages in your own memory to prevent duplicate processing. - old_messages = self._rc.memory.get() + old_messages = [] if ignore_memory else self._rc.memory.get() self._rc.memory.add_batch(news) # Filter out messages of interest. self._rc.news = [n for n in news if n.cause_by in self._rc.watch and n not in old_messages] From 5c149efee77c6a3c90382de7221f1370eab7d94c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 30 Nov 2023 19:33:27 +0800 Subject: [PATCH 133/135] fixbug: The assumption that messages in 'memory' have been processed has been revoked. --- metagpt/roles/qa_engineer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 763ab6a3f..de09cc4f0 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -167,3 +167,6 @@ class QaEngineer(Role): sent_from=self.profile, send_to=MESSAGE_ROUTE_TO_NONE, ) + + async def _observe(self, ignore_memory=False) -> int: + return await super(QaEngineer, self)._observe(ignore_memory=True) From 053eac62bcd990b748a4ce4578345880d882b276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Dec 2023 13:01:43 +0800 Subject: [PATCH 134/135] feat: +annotation --- metagpt/roles/qa_engineer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index de09cc4f0..f2e011ffd 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -169,4 +169,6 @@ class QaEngineer(Role): ) async def _observe(self, ignore_memory=False) -> int: + # This role has events that trigger and execute themselves based on conditions, and cannot rely on the + # content of memory to activate. return await super(QaEngineer, self)._observe(ignore_memory=True) From 4845dafb94966a502f153a8e5d223b19f60be2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Dec 2023 10:31:02 +0800 Subject: [PATCH 135/135] feat: +log --- tests/metagpt/test_gpt.py | 8 ++++++++ tests/metagpt/test_llm.py | 3 +++ 2 files changed, 11 insertions(+) diff --git a/tests/metagpt/test_gpt.py b/tests/metagpt/test_gpt.py index 285e8134c..431858d4c 100644 --- a/tests/metagpt/test_gpt.py +++ b/tests/metagpt/test_gpt.py @@ -15,6 +15,7 @@ from metagpt.logs import logger class TestGPT: def test_llm_api_ask(self, llm_api): answer = llm_api.ask("hello chatgpt") + logger.info(answer) assert len(answer) > 0 # def test_gptapi_ask_batch(self, llm_api): @@ -23,16 +24,19 @@ class TestGPT: def test_llm_api_ask_code(self, llm_api): answer = llm_api.ask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + logger.info(answer) assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_aask(self, llm_api): answer = await llm_api.aask("hello chatgpt") + logger.info(answer) assert len(answer) > 0 @pytest.mark.asyncio async def test_llm_api_aask_code(self, llm_api): answer = await llm_api.aask_code(["请扮演一个Google Python专家工程师，如果理解，回复明白", "写一个hello world"]) + logger.info(answer) assert len(answer) > 0 @pytest.mark.asyncio @@ -41,3 +45,7 @@ class TestGPT: costs = llm_api.get_costs() logger.info(costs) assert costs.total_cost > 0 + + +# if __name__ == "__main__": +# pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 03341212b..49969a2af 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -32,3 +32,6 @@ async def test_llm_acompletion(llm): assert len(await llm.acompletion(hello_msg)) > 0 assert len(await llm.acompletion_batch([hello_msg])) > 0 assert len(await llm.acompletion_batch_text([hello_msg])) > 0 + +# if __name__ == "__main__": +# pytest.main([__file__, "-s"])