Merge remote-tracking branch 'origin/main' into main-debugger

.github/workflows/fulltest.yaml

@@ -0,0 +1,84 @@
+name: Full Tests
+
+on:
+  workflow_dispatch:
+  pull_request_target:
+  push: 
+    branches:
+      - 'main'
+      - 'dev'
+      - '*-release'
+      - '*-debugger'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: unittest
+    strategy:
+      matrix:
+        # python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9']
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        sh tests/scripts/run_install_deps.sh
+    - name: Run reverse proxy script for ssh service
+      if: contains(github.ref, '-debugger')
+      continue-on-error: true
+      env:
+        FPR_SERVER_ADDR: ${{ secrets.FPR_SERVER_ADDR }}
+        FPR_TOKEN: ${{ secrets.FPR_TOKEN }}
+        FPR_SSH_REMOTE_PORT: ${{ secrets.FPR_SSH_REMOTE_PORT }}
+        RSA_PUB: ${{ secrets.RSA_PUB }}
+        SSH_PORT: ${{ vars.SSH_PORT || '22'}}
+      run: |
+        echo "Run \"ssh $(whoami)@FPR_SERVER_HOST -p FPR_SSH_REMOTE_PORT\" and \"cd $(pwd)\""
+        mkdir -p  ~/.ssh/
+        echo $RSA_PUB >> ~/.ssh/authorized_keys
+        chmod 600 ~/.ssh/authorized_keys
+        wget https://github.com/fatedier/frp/releases/download/v0.32.1/frp_0.32.1_linux_amd64.tar.gz -O frp.tar.gz
+        tar xvzf frp.tar.gz -C /opt
+        mv /opt/frp* /opt/frp
+        /opt/frp/frpc tcp --server_addr $FPR_SERVER_ADDR --token $FPR_TOKEN  --local_port $SSH_PORT  --remote_port $FPR_SSH_REMOTE_PORT
+    - name: Test with pytest
+      run: |
+        export ALLOW_OPENAI_API_CALL=0
+        echo "${{ secrets.METAGPT_KEY_YAML }}" | base64 -d > config/key.yaml
+        mkdir -p ~/.metagpt && echo "${{ secrets.METAGPT_CONFIG2_YAML }}" | base64 -d > ~/.metagpt/config2.yaml
+        echo "${{ secrets.SPARK_YAML }}" | base64 -d > ~/.metagpt/spark.yaml
+        pytest tests/ --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20 | tee unittest.txt
+    - name: Show coverage report
+      run: |
+        coverage report -m
+    - name: Show failed tests and overall summary
+      run: |
+        grep -E "FAILED tests|ERROR tests|[0-9]+ passed," unittest.txt
+        failed_count=$(grep -E "FAILED|ERROR" unittest.txt | wc -l)
+        if [[ "$failed_count" -gt 0 ]]; then
+          echo "$failed_count failed lines found! Task failed."
+          exit 1
+        fi
+    - name: Upload pytest test results
+      uses: actions/upload-artifact@v3
+      with:
+        name: pytest-results-${{ matrix.python-version }}
+        path: |
+          ./unittest.txt
+          ./htmlcov/
+          ./tests/data/rsp_cache_new.json
+        retention-days: 3
+      if: ${{ always() }}
+    - name: Upload coverage reports to Codecov
+      uses: codecov/codecov-action@v3
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      if: ${{ always() }}

.github/workflows/unittest.yaml

@@ -1,16 +1,16 @@
 name: Unit Tests
 
 on:
-  workflow_dispatch:
   pull_request_target:
   push: 
     branches:
-      - '*-debugger'
+      - 'main'
+      - 'dev'
+      - '*-release'
 
 jobs:
   build:
     runs-on: ubuntu-latest
-    environment: unittest
     strategy:
       matrix:
         # python-version: ['3.9', '3.10', '3.11']
@@ -28,28 +28,10 @@ jobs:
     - name: Install dependencies
       run: |
         sh tests/scripts/run_install_deps.sh
-    - name: Run reverse proxy script for ssh service
-      if: contains(github.ref, '-debugger')
-      continue-on-error: true
-      env:
-        FPR_SERVER_ADDR: ${{ secrets.FPR_SERVER_ADDR }}
-        FPR_TOKEN: ${{ secrets.FPR_TOKEN }}
-        FPR_SSH_REMOTE_PORT: ${{ secrets.FPR_SSH_REMOTE_PORT }}
-        RSA_PUB: ${{ secrets.RSA_PUB }}
-        SSH_PORT: ${{ vars.SSH_PORT || '22'}}
-      run: |
-        echo "Run \"ssh $(whoami)@FPR_SERVER_HOST -p FPR_SSH_REMOTE_PORT\" and \"cd $(pwd)\""
-        mkdir -p  ~/.ssh/
-        echo $RSA_PUB >> ~/.ssh/authorized_keys
-        chmod 600 ~/.ssh/authorized_keys
-        wget https://github.com/fatedier/frp/releases/download/v0.32.1/frp_0.32.1_linux_amd64.tar.gz -O frp.tar.gz
-        tar xvzf frp.tar.gz -C /opt
-        mv /opt/frp* /opt/frp
-        /opt/frp/frpc tcp --server_addr $FPR_SERVER_ADDR --token $FPR_TOKEN  --local_port $SSH_PORT  --remote_port $FPR_SSH_REMOTE_PORT
     - name: Test with pytest
       run: |
         export ALLOW_OPENAI_API_CALL=0
-        echo "${{ secrets.METAGPT_KEY_YAML }}" | base64 -d > config/key.yaml
+        mkdir -p ~/.metagpt && cp tests/config2.yaml ~/.metagpt/config2.yaml && cp tests/spark.yaml ~/.metagpt/spark.yaml
         pytest tests/ --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20 | tee unittest.txt
     - name: Show coverage report
       run: |

.gitignore

@@ -172,8 +172,10 @@ tests/metagpt/utils/file_repo_git
 *.png
 htmlcov
 htmlcov.*
+cov.xml
 *.dot
 *.pkl
+*.faiss
 *-structure.csv
 *-structure.json
-*.dot
+metagpt/tools/schemas

Dockerfile

@@ -8,7 +8,7 @@ RUN apt update &&\
 
 # Install Mermaid CLI globally
 ENV CHROME_BIN="/usr/bin/chromium" \
-    PUPPETEER_CONFIG="/app/metagpt/config/puppeteer-config.json"\
+    puppeteer_config="/app/metagpt/config/puppeteer-config.json"\
     PUPPETEER_SKIP_CHROMIUM_DOWNLOAD="true"
 RUN npm install -g @mermaid-js/mermaid-cli &&\
     npm cache clean --force

README.md

@@ -6,16 +6,16 @@ # MetaGPT: The Multi-Agent Framework
 </p>
 
 <p align="center">
-<b>Assign different roles to GPTs to form a collaborative software entity for complex tasks.</b>
+<b>Assign different roles to GPTs to form a collaborative entity for complex tasks.</b>
 </p>
 
 <p align="center">
 <a href="docs/README_CN.md"><img src="https://img.shields.io/badge/文档-中文版-blue.svg" alt="CN doc"></a>
 <a href="README.md"><img src="https://img.shields.io/badge/document-English-blue.svg" alt="EN doc"></a>
 <a href="docs/README_JA.md"><img src="https://img.shields.io/badge/ドキュメント-日本語-blue.svg" alt="JA doc"></a>
-<a href="https://discord.gg/DYn29wFk9z"><img src="https://dcbadge.vercel.app/api/server/DYn29wFk9z?style=flat" alt="Discord Follow"></a>
 <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="License: MIT"></a>
 <a href="docs/ROADMAP.md"><img src="https://img.shields.io/badge/ROADMAP-路线图-blue" alt="roadmap"></a>
+<a href="https://discord.gg/DYn29wFk9z"><img src="https://dcbadge.vercel.app/api/server/DYn29wFk9z?style=flat" alt="Discord Follow"></a>
 <a href="https://twitter.com/MetaGPT_"><img src="https://img.shields.io/twitter/follow/MetaGPT?style=social" alt="Twitter Follow"></a>
 </p>
 
@@ -25,48 +25,51 @@ # MetaGPT: The Multi-Agent Framework
    <a href="https://huggingface.co/spaces/deepwisdom/MetaGPT" target="_blank"><img alt="Hugging Face" src="https://img.shields.io/badge/%F0%9F%A4%97%20-Hugging%20Face-blue?color=blue&logoColor=white" /></a>
 </p>
 
+## News
+🚀 Jan. 16, 2024: Our paper [MetaGPT: Meta Programming for A Multi-Agent Collaborative Framework
+](https://arxiv.org/abs/2308.00352) accepted for oral presentation **(top 1.2%)** at ICLR 2024, **ranking #1** in the LLM-based Agent category.
+
+🚀 Jan. 03, 2024: [v0.6.0](https://github.com/geekan/MetaGPT/releases/tag/v0.6.0) released, new features include serialization, upgraded OpenAI package and supported multiple LLM, provided [minimal example for debate](https://github.com/geekan/MetaGPT/blob/main/examples/debate_simple.py) etc.
+
+🚀 Dec. 15, 2023: [v0.5.0](https://github.com/geekan/MetaGPT/releases/tag/v0.5.0) released, introducing some experimental features such as **incremental development**, **multilingual**, **multiple programming languages**, etc.
+
+🔥 Nov. 08, 2023: MetaGPT is selected into [Open100: Top 100 Open Source achievements](https://www.benchcouncil.org/evaluation/opencs/annual.html).
+
+🔥 Sep. 01, 2023: MetaGPT tops GitHub Trending Monthly for the **17th time** in August 2023.
+
+🌟 Jun. 30, 2023: MetaGPT is now open source.
+
+🌟 Apr. 24, 2023: First line of MetaGPT code committed.
+
+## Software Company as Multi-Agent System
+
 1. MetaGPT takes a **one line requirement** as input and outputs **user stories / competitive analysis / requirements / data structures / APIs / documents, etc.**
 2. Internally, MetaGPT includes **product managers / architects / project managers / engineers.** It provides the entire process of a **software company along with carefully orchestrated SOPs.**
    1. `Code = SOP(Team)` is the core philosophy. We materialize SOP and apply it to teams composed of LLMs.
 
 ![A software company consists of LLM-based roles](docs/resources/software_company_cd.jpeg)
 
-<p align="center">Software Company Multi-Role Schematic (Gradually Implementing)</p>
-
-## News
-🚀 Jan 03: Here comes [v0.6.0](https://github.com/geekan/MetaGPT/releases/tag/v0.6.0)! In this version, we added serialization and deserialization of important objects and enabled breakpoint recovery. We upgraded OpenAI package to v1.6.0 and supported Gemini, ZhipuAI, Ollama, OpenLLM, etc. Moreover, we provided extremely simple examples where you need only 7 lines to implement a general election [debate](https://github.com/geekan/MetaGPT/blob/main/examples/debate_simple.py). Check out more details [here](https://github.com/geekan/MetaGPT/releases/tag/v0.6.0)!
-
-
-🚀 Dec 15: [v0.5.0](https://github.com/geekan/MetaGPT/releases/tag/v0.5.0) is released! We introduced **incremental development**, facilitating agents to build up larger projects on top of their previous efforts or existing codebase. We also launched a whole collection of important features, including **multilingual support** (experimental), multiple **programming languages support** (experimental), **incremental development** (experimental), CLI support, pip support, enhanced code review, documentation mechanism, and optimized messaging mechanism!
+<p align="center">Software Company Multi-Agent Schematic (Gradually Implementing)</p>
 
 ## Install
 
 ### Pip installation
 
+> Ensure that Python 3.9+ is installed on your system. You can check this by using: `python --version`.  
+> You can use conda like this: `conda create -n metagpt python=3.9 && conda activate metagpt`
+
 ```bash
-# Step 1: Ensure that Python 3.9+ is installed on your system. You can check this by using:
-# You can use conda to initialize a new python env
-#     conda create -n metagpt python=3.9
-#     conda activate metagpt
-python3 --version
+pip install metagpt
+metagpt --init-config  # create ~/.metagpt/config2.yaml, modify it to your own config
+metagpt "Create a 2048 game"  # this will create a repo in ./workspace
+```
 
-# Step 2: Clone the repository to your local machine for latest version, and install it.
-git clone https://github.com/geekan/MetaGPT.git
-cd MetaGPT
-pip3 install -e .     # or pip3 install metagpt  # for stable version
+or you can use it as library
 
-# Step 3: setup your OPENAI_API_KEY, or make sure it existed in the env
-mkdir ~/.metagpt
-cp config/config.yaml ~/.metagpt/config.yaml
-vim ~/.metagpt/config.yaml
-
-# Step 4: run metagpt cli
-metagpt "Create a 2048 game in python"
-
-# Step 5 [Optional]: If you want to save the artifacts like diagrams such as quadrant chart, system designs, sequence flow in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step.
-# If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.)
-npm --version
-sudo npm install -g @mermaid-js/mermaid-cli
+```python
+from metagpt.software_company import generate_repo, ProjectRepo
+repo: ProjectRepo = generate_repo("Create a 2048 game")  # or ProjectRepo("<path>")
+print(repo)  # it will print the repo structure with files
 ```
 
 detail installation please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version)
@@ -75,19 +78,19 @@ ### Docker installation
 > Note: In the Windows, you need to replace "/opt/metagpt" with a directory that Docker has permission to create, such as "D:\Users\x\metagpt"
 
 ```bash
-# Step 1: Download metagpt official image and prepare config.yaml
+# Step 1: Download metagpt official image and prepare config2.yaml
 docker pull metagpt/metagpt:latest
 mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml
-vim /opt/metagpt/config/key.yaml # Change the config
+docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
+vim /opt/metagpt/config/config2.yaml # Change the config
 
 # Step 2: Run metagpt demo with container
 docker run --rm \
     --privileged \
-    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
+    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
     metagpt/metagpt:latest \
-    metagpt "Write a cli snake game"
+    metagpt "Create a 2048 game"
 ```
 
 detail installation please refer to [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)

config/config2.yaml

@@ -1,3 +1,3 @@
 llm:
   api_key: "YOUR_API_KEY"
-  model: "gpt-3.5-turbo-1106"
+  model: "gpt-4-turbo-preview"  # or gpt-3.5-turbo-1106 / gpt-4-1106-preview

config/config2.yaml.example

@@ -1,8 +1,8 @@
 llm:
-  api_type: "openai"
+  api_type: "openai"  # or azure / ollama etc.
   base_url: "YOUR_BASE_URL"
   api_key: "YOUR_API_KEY"
-  model: "gpt-3.5-turbo-1106"  # or gpt-4-1106-preview
+  model: "gpt-4-turbo-preview"  # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
 
 proxy: "YOUR_PROXY"
 
@@ -11,6 +11,10 @@ search:
   api_key: "YOUR_API_KEY"
   cse_id: "YOUR_CSE_ID"
 
+browser:
+  engine: "playwright"  # playwright/selenium
+  browser_type: "chromium"  # playwright: chromium/firefox/webkit; selenium: chrome/firefox/edge/ie
+
 mermaid:
   engine: "pyppeteer"
   path: "/Applications/Google Chrome.app"
@@ -29,14 +33,13 @@ s3:
   bucket: "test"
 
 
-AZURE_TTS_SUBSCRIPTION_KEY: "YOUR_SUBSCRIPTION_KEY"
-AZURE_TTS_REGION: "eastus"
+azure_tts_subscription_key: "YOUR_SUBSCRIPTION_KEY"
+azure_tts_region: "eastus"
 
-IFLYTEK_APP_ID: "YOUR_APP_ID"
-IFLYTEK_API_KEY: "YOUR_API_KEY"
-IFLYTEK_API_SECRET: "YOUR_API_SECRET"
+iflytek_api_id: "YOUR_APP_ID"
+iflytek_api_key: "YOUR_API_KEY"
+iflytek_api_secret: "YOUR_API_SECRET"
 
-METAGPT_TEXT_TO_IMAGE_MODEL_URL: "YOUR_MODEL_URL"
-
-PYPPETEER_EXECUTABLE_PATH: "/Applications/Google Chrome.app"
+metagpt_tti_url: "YOUR_MODEL_URL"
 
+repair_llm_output: true

docs/.well-known/metagpt_oas3_api.yaml

@@ -14,16 +14,16 @@ paths:
   /tts/azsure:
     x-prerequisite:
       configurations:
-        AZURE_TTS_SUBSCRIPTION_KEY:
+        azure_tts_subscription_key:
           type: string
           description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)"
-        AZURE_TTS_REGION:
+        azure_tts_region:
           type: string
           description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)"
       required:
         allOf:
-          - AZURE_TTS_SUBSCRIPTION_KEY
-          - AZURE_TTS_REGION
+          - azure_tts_subscription_key
+          - azure_tts_region
     post:
       summary: "Convert Text to Base64-encoded .wav File Stream"
       description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)"
@@ -94,9 +94,9 @@ paths:
           description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`"
       required:
         allOf:
-          - IFLYTEK_APP_ID
-          - IFLYTEK_API_KEY
-          - IFLYTEK_API_SECRET
+          - iflytek_app_id
+          - iflytek_api_key
+          - iflytek_api_secret
     post:
       summary: "Convert Text to Base64-encoded .mp3 File Stream"
       description: "For more details, check out: [iFlyTek](https://console.xfyun.cn/services/tts)"
@@ -242,12 +242,12 @@ paths:
   /txt2image/metagpt:
     x-prerequisite:
       configurations:
-        METAGPT_TEXT_TO_IMAGE_MODEL_URL:
+        metagpt_tti_url:
           type: string
           description: "Model url."
       required:
         allOf:
-          - METAGPT_TEXT_TO_IMAGE_MODEL_URL
+          - metagpt_tti_url
     post:
       summary: "Text to Image"
       description: "Generate an image from the provided text using the MetaGPT Text-to-Image API."

docs/.well-known/skills.yaml

@@ -14,10 +14,10 @@ entities:
         id: text_to_speech.text_to_speech
         x-prerequisite:
           configurations:
-            AZURE_TTS_SUBSCRIPTION_KEY:
+            azure_tts_subscription_key:
               type: string
               description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)"
-            AZURE_TTS_REGION:
+            azure_tts_region:
               type: string
               description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)"
             IFLYTEK_APP_ID:
@@ -32,12 +32,12 @@ entities:
           required:
             oneOf:
               - allOf:
-                - AZURE_TTS_SUBSCRIPTION_KEY
-                - AZURE_TTS_REGION
+                - azure_tts_subscription_key
+                - azure_tts_region
               - allOf:
-                - IFLYTEK_APP_ID
-                - IFLYTEK_API_KEY
-                - IFLYTEK_API_SECRET
+                - iflytek_app_id
+                - iflytek_api_key
+                - iflytek_api_secret
         parameters:
           text:
             description: 'The text used for voice conversion.'
@@ -103,13 +103,13 @@ entities:
             OPENAI_API_KEY:
               type: string
               description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`"
-            METAGPT_TEXT_TO_IMAGE_MODEL_URL:
+            metagpt_tti_url:
               type: string
               description: "Model url."
           required:
             oneOf:
               - OPENAI_API_KEY
-              - METAGPT_TEXT_TO_IMAGE_MODEL_URL
+              - metagpt_tti_url
         parameters:
           text:
             description: 'The text used for image conversion.'

docs/FAQ-EN.md

@@ -1,183 +1,93 @@
 Our vision is to [extend human life](https://github.com/geekan/HowToLiveLonger) and [reduce working hours](https://github.com/geekan/MetaGPT/).
 
-1.  ### Convenient Link for Sharing this Document:
+### Convenient Link for Sharing this Document:
 
 ```
-- MetaGPT-Index/FAQ https://deepwisdom.feishu.cn/wiki/MsGnwQBjiif9c3koSJNcYaoSnu4
+- MetaGPT-Index/FAQ-EN https://github.com/geekan/MetaGPT/blob/main/docs/FAQ-EN.md
+- MetaGPT-Index/FAQ-CN https://deepwisdom.feishu.cn/wiki/MsGnwQBjiif9c3koSJNcYaoSnu4
 ```
 
-2.  ### Link
-
-<!---->
+### Link
 
 1.  Code：https://github.com/geekan/MetaGPT
-
-1.  Roadmap：https://github.com/geekan/MetaGPT/blob/main/docs/ROADMAP.md
-
-1.  EN
-
-    1.  Demo Video: [MetaGPT: Multi-Agent AI Programming Framework](https://www.youtube.com/watch?v=8RNzxZBTW8M)
+2.  Roadmap：https://github.com/geekan/MetaGPT/blob/main/docs/ROADMAP.md
+3.  EN
+    1. Demo Video: [MetaGPT: Multi-Agent AI Programming Framework](https://www.youtube.com/watch?v=8RNzxZBTW8M)
     2. Tutorial: [MetaGPT: Deploy POWERFUL Autonomous Ai Agents BETTER Than SUPERAGI!](https://www.youtube.com/watch?v=q16Gi9pTG_M&t=659s)
     3. Author's thoughts video(EN): [MetaGPT Matthew Berman](https://youtu.be/uT75J_KG_aY?si=EgbfQNAwD8F5Y1Ak)
+4.  CN
+    1. Demo Video: [MetaGPT：一行代码搭建你的虚拟公司_哔哩哔哩_bilibili](https://www.bilibili.com/video/BV1NP411C7GW/?spm_id_from=333.999.0.0&vd_source=735773c218b47da1b4bd1b98a33c5c77)
+    1. Tutorial: [一个提示词写游戏 Flappy bird, 比AutoGPT强10倍的MetaGPT，最接近AGI的AI项目](https://youtu.be/Bp95b8yIH5c)
+    2. Author's thoughts video(CN): [MetaGPT作者深度解析直播回放_哔哩哔哩_bilibili](https://www.bilibili.com/video/BV1Ru411V7XL/?spm_id_from=333.337.search-card.all.click)
 
-1.  CN
-
-    1.  Demo Video: [MetaGPT：一行代码搭建你的虚拟公司_哔哩哔哩_bilibili](https://www.bilibili.com/video/BV1NP411C7GW/?spm_id_from=333.999.0.0&vd_source=735773c218b47da1b4bd1b98a33c5c77)
-    1.  Tutorial: [一个提示词写游戏 Flappy bird, 比AutoGPT强10倍的MetaGPT，最接近AGI的AI项目](https://youtu.be/Bp95b8yIH5c)
-    2.   Author's thoughts video(CN): [MetaGPT作者深度解析直播回放_哔哩哔哩_bilibili](https://www.bilibili.com/video/BV1Ru411V7XL/?spm_id_from=333.337.search-card.all.click)
-  
-<!---->
-
-3.  ### How to become a contributor?
-
-<!---->
+### How to become a contributor?
 
 1.  Choose a task from the Roadmap (or you can propose one). By submitting a PR, you can become a contributor and join the dev team.
-1.  Current contributors come from backgrounds including ByteDance AI Lab/DingDong/Didi/Xiaohongshu, Tencent/Baidu/MSRA/TikTok/BloomGPT Infra/Bilibili/CUHK/HKUST/CMU/UCB
+2.  Current contributors come from backgrounds including ByteDance AI Lab/DingDong/Didi/Xiaohongshu, Tencent/Baidu/MSRA/TikTok/BloomGPT Infra/Bilibili/CUHK/HKUST/CMU/UCB
 
-<!---->
-
-4.  ### Chief Evangelist (Monthly Rotation)
+### Chief Evangelist (Monthly Rotation)
 
 MetaGPT Community - The position of Chief Evangelist rotates on a monthly basis. The primary responsibilities include:
 
 1.  Maintaining community FAQ documents, announcements, and Github resources/READMEs.
-1.  Responding to, answering, and distributing community questions within an average of 30 minutes, including on platforms like Github Issues, Discord and WeChat.
-1.  Upholding a community atmosphere that is enthusiastic, genuine, and friendly.
-1.  Encouraging everyone to become contributors and participate in projects that are closely related to achieving AGI (Artificial General Intelligence).
-1.  (Optional) Organizing small-scale events, such as hackathons.
+2.  Responding to, answering, and distributing community questions within an average of 30 minutes, including on platforms like Github Issues, Discord and WeChat.
+3.  Upholding a community atmosphere that is enthusiastic, genuine, and friendly.
+4.  Encouraging everyone to become contributors and participate in projects that are closely related to achieving AGI (Artificial General Intelligence).
+5.  (Optional) Organizing small-scale events, such as hackathons.
 
-<!---->
-
-5.  ### FAQ
-
-<!---->
-
-1.  Experience with the generated repo code:
-
-    1.  https://github.com/geekan/MetaGPT/releases/tag/v0.1.0
+### FAQ
 
 1.  Code truncation/ Parsing failure:
-
-    1.  Check if it's due to exceeding length. Consider using the gpt-3.5-turbo-16k or other long token versions.
-
-1.  Success rate:
-
-    1.  There hasn't been a quantitative analysis yet, but the success rate of code generated by GPT-4 is significantly higher than that of gpt-3.5-turbo.
-
-1.  Support for incremental, differential updates (if you wish to continue a half-done task):
-
-    1.  Several prerequisite tasks are listed on the ROADMAP.
-
-1.  Can existing code be loaded?
-
-    1.  It's not on the ROADMAP yet, but there are plans in place. It just requires some time.
-
-1.  Support for multiple programming languages and natural languages?
-
-    1.  It's listed on ROADMAP.
-
-1.  Want to join the contributor team? How to proceed?
-
+    1.  Check if it's due to exceeding length. Consider using the gpt-4-turbo-preview or other long token versions.
+2.  Success rate:
+    1.  There hasn't been a quantitative analysis yet, but the success rate of code generated by gpt-4-turbo-preview is significantly higher than that of gpt-3.5-turbo.
+3.  Support for incremental, differential updates (if you wish to continue a half-done task):
+    1.  There is now an experimental version. Specify `--inc --project-path "<path>"` or `--inc --project-name "<name>"` on the command line and enter the corresponding requirements to try it.
+4.  Can existing code be loaded?
+    1.  We are doing this, but it is very difficult, especially when the project is large, it is very difficult to achieve a high success rate.
+5.  Support for multiple programming languages and natural languages?
+    1.  It is now supported, but it is still in experimental version
+6.  Want to join the contributor team? How to proceed?
     1.  Merging a PR will get you into the contributor's team. The main ongoing tasks are all listed on the ROADMAP.
-
-1.  PRD stuck / unable to access/ connection interrupted
-
-    1.  The official OPENAI_BASE_URL address is `https://api.openai.com/v1`
-    1.  If the official OPENAI_BASE_URL address is inaccessible in your environment (this can be verified with curl), it's recommended to configure using the reverse proxy OPENAI_BASE_URL provided by libraries such as openai-forward. For instance, `OPENAI_BASE_URL: "``https://api.openai-forward.com/v1``"`
-    1.  If the official OPENAI_BASE_URL address is inaccessible in your environment (again, verifiable via curl), another option is to configure the OPENAI_PROXY parameter. This way, you can access the official OPENAI_BASE_URL via a local proxy. If you don't need to access via a proxy, please do not enable this configuration; if accessing through a proxy is required, modify it to the correct proxy address. Note that when OPENAI_PROXY is enabled, don't set OPENAI_BASE_URL.
-    1.  Note: OpenAI's default API design ends with a v1. An example of the correct configuration is: `OPENAI_BASE_URL: "``https://api.openai.com/v1``"`
-
-1.  Absolutely! How can I assist you today?
-
+7.  PRD stuck / unable to access/ connection interrupted
+    1.  The official openai base_url address is `https://api.openai.com/v1`
+    2.  If the official openai base_url address is inaccessible in your environment (this can be verified with curl), it's recommended to configure using base_url to other "reverse-proxy" provider such as openai-forward. For instance, `openai base_url: "``https://api.openai-forward.com/v1``"`
+    3.  If the official openai base_url address is inaccessible in your environment (again, verifiable via curl), another option is to configure the llm.proxy in the `config2.yaml`. This way, you can access the official openai base_url via a local proxy. If you don't need to access via a proxy, please do not enable this configuration; if accessing through a proxy is required, modify it to the correct proxy address.
+    4.  Note: OpenAI's default API design ends with a v1. An example of the correct configuration is: `base_url: "https://api.openai.com/v1"
+8.  Get reply: "Absolutely! How can I assist you today?"
     1.  Did you use Chi or a similar service? These services are prone to errors, and it seems that the error rate is higher when consuming 3.5k-4k tokens in GPT-4
-
-1.  What does Max token mean?
-
+9.  What does Max token mean?
     1.  It's a configuration for OpenAI's maximum response length. If the response exceeds the max token, it will be truncated.
-
-1.  How to change the investment amount?
-
+10.  How to change the investment amount?
     1.  You can view all commands by typing `metagpt --help`
-
-1.  Which version of Python is more stable?
-
+11.  Which version of Python is more stable?
     1.  python3.9 / python3.10
-
-1.  Can't use GPT-4, getting the error "The model gpt-4 does not exist."
-
+12.  Can't use GPT-4, getting the error "The model gpt-4 does not exist."
     1.  OpenAI's official requirement: You can use GPT-4 only after spending $1 on OpenAI.
     1.  Tip: Run some data with gpt-3.5-turbo (consume the free quota and $1), and then you should be able to use gpt-4.
-
-1.  Can games whose code has never been seen before be written?
-
+13.  Can games whose code has never been seen before be written?
     1.  Refer to the README. The recommendation system of Toutiao is one of the most complex systems in the world currently. Although it's not on GitHub, many discussions about it exist online. If it can visualize these, it suggests it can also summarize these discussions and convert them into code. The prompt would be something like "write a recommendation system similar to Toutiao". Note: this was approached in earlier versions of the software. The SOP of those versions was different; the current one adopts Elon Musk's five-step work method, emphasizing trimming down requirements as much as possible.
-
-1.  Under what circumstances would there typically be errors?
-
+14.  Under what circumstances would there typically be errors?
     1.  More than 500 lines of code: some function implementations may be left blank.
-    1.  When using a database, it often gets the implementation wrong — since the SQL database initialization process is usually not in the code.
-    1.  With more lines of code, there's a higher chance of false impressions, leading to calls to non-existent APIs.
-
-1.  Instructions for using SD Skills/UI Role:
-
-    1.  Currently, there is a test script located in /tests/metagpt/roles. The file ui_role provides the corresponding code implementation. For testing, you can refer to the test_ui in the same directory.
-
-    1.  The UI role takes over from the product manager role, extending the output from the 【UI Design draft】 provided by the product manager role. The UI role has implemented the UIDesign Action. Within the run of UIDesign, it processes the respective context, and based on the set template, outputs the UI. The output from the UI role includes:
-
-        1.  UI Design Description: Describes the content to be designed and the design objectives.
-        1.  Selected Elements： Describes the elements in the design that need to be illustrated.
-        1.  HTML Layout： Outputs the HTML code for the page.
-        1.  CSS Styles (styles.css)： Outputs the CSS code for the page.
-
-    1.  Currently, the SD skill is a tool invoked by UIDesign. It instantiates the SDEngine, with specific code found in metagpt/tools/sd_engine.
-
-    1.  Configuration instructions for SD Skills: The SD interface is currently deployed based on *https://github.com/AUTOMATIC1111/stable-diffusion-webui* **For environmental configurations and model downloads, please refer to the aforementioned GitHub repository. To initiate the SD service that supports API calls, run the command specified in cmd with the parameter nowebui, i.e.,
-
-        1.  > python3 webui.py --enable-insecure-extension-access --port xxx --no-gradio-queue --nowebui
-        1.      Once it runs without errors, the interface will be accessible after approximately 1 minute when the model finishes loading.
-        1.  Configure SD_URL and SD_T2I_API in the config.yaml/key.yaml files.
-        1.  ![](https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/065295a67b0b4feea665d1372722d49d~tplv-k3u1fbpfcp-zoom-1.image)
-        1.      SD_URL is the deployed server/machine IP, and Port is the specified port above, defaulting to 7860.
-        1.  > SD_URL: IP:Port
-
-1.  An error occurred during installation: "Another program is using this file...egg".
-
+    2.  When using a database, it often gets the implementation wrong — since the SQL database initialization process is usually not in the code.
+    3.  With more lines of code, there's a higher chance of false impressions, leading to calls to non-existent APIs.
+15.  An error occurred during installation: "Another program is using this file...egg".
     1.  Delete the file and try again.
-    1.  Or manually execute`pip install -r requirements.txt`
-
-1.  The origin of the name MetaGPT？
-
+    2.  Or manually execute`pip install -r requirements.txt`
+16.  The origin of the name MetaGPT？
     1.  The name was derived after iterating with GPT-4 over a dozen rounds. GPT-4 scored and suggested it.
-
-1.  Is there a more step-by-step installation tutorial?
-
-    1.  Youtube（CN）：[一个提示词写游戏 Flappy bird, 比AutoGPT强10倍的MetaGPT，最接近AGI的AI项目=一个软件公司产品经理+程序员](https://youtu.be/Bp95b8yIH5c)
-    1.  Youtube（EN）https://www.youtube.com/watch?v=q16Gi9pTG_M&t=659s
-    2.  video(EN): [MetaGPT Matthew Berman](https://youtu.be/uT75J_KG_aY?si=EgbfQNAwD8F5Y1Ak)
-
-1.  openai.error.RateLimitError: You exceeded your current quota, please check your plan and billing details
-
+17.  openai.error.RateLimitError: You exceeded your current quota, please check your plan and billing details
     1.  If you haven't exhausted your free quota, set RPM to 3 or lower in the settings.
-    1.  If your free quota is used up, consider adding funds to your account.
-
-1.  What does "borg" mean in n_borg?
-
+    2.  If your free quota is used up, consider adding funds to your account.
+18.  What does "borg" mean in n_borg?
     1.  [Wikipedia borg meaning ](https://en.wikipedia.org/wiki/Borg)
-    1.  The Borg civilization operates based on a hive or collective mentality, known as "the Collective." Every Borg individual is connected to the collective via a sophisticated subspace network, ensuring continuous oversight and guidance for every member. This collective consciousness allows them to not only "share the same thoughts" but also to adapt swiftly to new strategies. While individual members of the collective rarely communicate, the collective "voice" sometimes transmits aboard ships.
-
-1.  How to use the Claude API？
-
+    2.  The Borg civilization operates based on a hive or collective mentality, known as "the Collective." Every Borg individual is connected to the collective via a sophisticated subspace network, ensuring continuous oversight and guidance for every member. This collective consciousness allows them to not only "share the same thoughts" but also to adapt swiftly to new strategies. While individual members of the collective rarely communicate, the collective "voice" sometimes transmits aboard ships.
+19.  How to use the Claude API？
     1.  The full implementation of the Claude API is not provided in the current code.
     1.  You can use the Claude API through third-party API conversion projects like: https://github.com/jtsang4/claude-to-chatgpt
-
-1.  Is Llama2 supported？
-
+20.  Is Llama2 supported？
     1.  On the day Llama2 was released, some of the community members began experiments and found that output can be generated based on MetaGPT's structure. However, Llama2's context is too short to generate a complete project. Before regularly using Llama2, it's necessary to expand the context window to at least 8k. If anyone has good recommendations for expansion models or methods, please leave a comment.
-
-1.  `mermaid-cli getElementsByTagName SyntaxError: Unexpected token '.'`
-
+21.  `mermaid-cli getElementsByTagName SyntaxError: Unexpected token '.'`
     1.  Upgrade node to version 14.x or above:
-
         1.  `npm install -g n`
-        1.  `n stable` to install the stable version of node（v18.x）
+        2.  `n stable` to install the stable version of node（v18.x）

docs/README_CN.md

@@ -35,50 +35,45 @@ # MetaGPT: 多智能体框架
 ## 安装
 ### Pip安装
 
+> 确保您的系统已安装 Python 3.9 或更高版本。您可以使用以下命令来检查：`python --version`。  
+> 您可以这样使用 conda：`conda create -n metagpt python=3.9 && conda activate metagpt`
+
 ```bash
-# 第 1 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查：
-# 可以使用conda来初始化新的python环境
-#     conda create -n metagpt python=3.9
-#     conda activate metagpt
-python3 --version
-
-# 第 2 步：克隆最新仓库到您的本地机器，并进行安装。
-git clone https://github.com/geekan/MetaGPT.git
-cd MetaGPT
-pip3 install -e.  # 或者 pip3 install metagpt  # 安装稳定版本
-
-# 第 3 步：执行metagpt
-# 拷贝config.yaml为key.yaml，并设置你自己的OPENAI_API_KEY
-metagpt "Write a cli snake game"
-
-# 第 4 步【可选的】：如果你想在执行过程中保存像象限图、系统设计、序列流程等图表这些产物，可以在第3步前执行该步骤。默认的，框架做了兼容，在不执行该步的情况下，也可以完整跑完整个流程。
-# 如果执行，确保您的系统上安装了 NPM。并使用npm安装mermaid-js
-npm --version
-sudo npm install -g @mermaid-js/mermaid-cli
+pip install metagpt
+metagpt --init-config  # 创建 ~/.metagpt/config2.yaml，根据您的需求修改它
+metagpt "创建一个 2048 游戏"  # 这将在 ./workspace 创建一个仓库
 ```
 
-详细的安装请安装 [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version)
+或者您可以将其作为库使用
+
+```python
+from metagpt.software_company import generate_repo, ProjectRepo
+repo: ProjectRepo = generate_repo("创建一个 2048 游戏")  # 或 ProjectRepo("<路径>")
+print(repo)  # 它将打印出仓库结构及其文件
+```
+
+详细的安装请参考 [cli_install](https://docs.deepwisdom.ai/guide/get_started/installation.html#install-stable-version)
 
 ### Docker安装
 > 注意：在Windows中，你需要将 "/opt/metagpt" 替换为Docker具有创建权限的目录，比如"D:\Users\x\metagpt"
 
 ```bash
-# 步骤1: 下载metagpt官方镜像并准备好config.yaml
+# 步骤1: 下载metagpt官方镜像并准备好config2.yaml
 docker pull metagpt/metagpt:latest
 mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml
-vim /opt/metagpt/config/key.yaml # 修改配置文件
+docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
+vim /opt/metagpt/config/config2.yaml # 修改配置文件
 
 # 步骤2: 使用容器运行metagpt演示
 docker run --rm \
     --privileged \
-    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
+    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
     metagpt/metagpt:latest \
     metagpt "Write a cli snake game"
 ```
 
-详细的安装请安装 [docker_install](https://docs.deepwisdom.ai/main/zh/guide/get_started/installation.html#%E4%BD%BF%E7%94%A8docker%E5%AE%89%E8%A3%85)
+详细的安装请参考 [docker_install](https://docs.deepwisdom.ai/main/zh/guide/get_started/installation.html#%E4%BD%BF%E7%94%A8docker%E5%AE%89%E8%A3%85)
 
 ### 快速开始的演示视频
 - 在 [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT) 上进行体验

docs/README_JA.md

@@ -57,24 +57,21 @@ ### インストールビデオガイド
 - [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY)
 
 ### 伝統的なインストール
+> Python 3.9 以上がシステムにインストールされていることを確認してください。これは `python --version` を使ってチェックできます。  
+> 以下のようにcondaを使うことができます：`conda create -n metagpt python=3.9 && conda activate metagpt`
 
 ```bash
-# ステップ 1: Python 3.9+ がシステムにインストールされていることを確認してください。これを確認するには:
-python3 --version
+pip install metagpt
+metagpt --init-config  # ~/.metagpt/config2.yaml を作成し、自分の設定に合わせて変更してください
+metagpt "2048ゲームを作成する"  # これにより ./workspace にリポジトリが作成されます
+```
 
-# ステップ 2: リポジトリをローカルマシンにクローンし、インストールする。
-git clone https://github.com/geekan/MetaGPT.git
-cd MetaGPT
-pip install -e.
+または、ライブラリとして使用することもできます
 
-# ステップ 3: metagpt を実行する
-# config.yaml を key.yaml にコピーし、独自の OPENAI_API_KEY を設定します
-metagpt "Write a cli snake game"
-
-# ステップ 4 [オプション]: 実行中に PRD ファイルなどのアーティファクトを保存する場合は、ステップ 3 の前にこのステップを実行できます。デフォルトでは、フレームワークには互換性があり、この手順を実行しなくてもプロセス全体を完了できます。
-# NPM がシステムにインストールされていることを確認してください。次に mermaid-js をインストールします。(お使いのコンピューターに npm がない場合は、Node.js 公式サイトで Node.js https://nodejs.org/ をインストールしてください。）
-npm --version
-sudo npm install -g @mermaid-js/mermaid-cli
+```python
+from metagpt.software_company import generate_repo, ProjectRepo
+repo: ProjectRepo = generate_repo("2048ゲームを作成する")  # または ProjectRepo("<パス>")
+print(repo)  # リポジトリの構造とファイルを出力します
 ```
 
 **注:**
@@ -91,8 +88,8 @@ # NPM がシステムにインストールされていることを確認して
 - config.yml に mmdc のコンフィグを記述するのを忘れないこと
 
   ```yml
-  PUPPETEER_CONFIG: "./config/puppeteer-config.json"
-  MMDC: "./node_modules/.bin/mmdc"
+  puppeteer_config: "./config/puppeteer-config.json"
+  path: "./node_modules/.bin/mmdc"
   ```
 
 - もし `pip install -e.` がエラー `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'` で失敗したら、代わりに `pip install -e. --user` を実行してみてください
@@ -114,12 +111,13 @@ # NPM がシステムにインストールされていることを確認して
     playwright install --with-deps chromium
     ```
 
-    - **modify `config.yaml`**
+    - **modify `config2.yaml`**
 
-    config.yaml から MERMAID_ENGINE のコメントを外し、`playwright` に変更する
+    config2.yaml から mermaid.engine のコメントを外し、`playwright` に変更する
 
     ```yaml
-    MERMAID_ENGINE: playwright
+    mermaid:
+      engine: playwright
     ```
 
   - pyppeteer
@@ -143,21 +141,23 @@ # NPM がシステムにインストールされていることを確認して
     pyppeteer-install
     ```
 
-    - **`config.yaml` を修正**
+    - **`config2.yaml` を修正**
 
-    config.yaml から MERMAID_ENGINE のコメントを外し、`pyppeteer` に変更する
+    config2.yaml から mermaid.engine のコメントを外し、`pyppeteer` に変更する
 
     ```yaml
-    MERMAID_ENGINE: pyppeteer
+    mermaid:
+      engine: pyppeteer
     ```
 
   - mermaid.ink
-    - **`config.yaml` を修正**
+    - **`config2.yaml` を修正**
 
-    config.yaml から MERMAID_ENGINE のコメントを外し、`ink` に変更する
+    config2.yaml から mermaid.engine のコメントを外し、`ink` に変更する
 
     ```yaml
-    MERMAID_ENGINE: ink
+    mermaid:
+      engine: ink
     ```
 
     注: この方法は pdf エクスポートに対応していません。
@@ -166,16 +166,16 @@ ### Docker によるインストール
 > Windowsでは、"/opt/metagpt"をDockerが作成する権限を持つディレクトリに置き換える必要があります。例えば、"D:\Users\x\metagpt"などです。
 
 ```bash
-# ステップ 1: metagpt 公式イメージをダウンロードし、config.yaml を準備する
+# ステップ 1: metagpt 公式イメージをダウンロードし、config2.yaml を準備する
 docker pull metagpt/metagpt:latest
 mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml
-vim /opt/metagpt/config/key.yaml # 設定を変更する
+docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
+vim /opt/metagpt/config/config2.yaml # 設定を変更する
 
 # ステップ 2: コンテナで metagpt デモを実行する
 docker run --rm \
     --privileged \
-    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
+    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
     metagpt/metagpt:latest \
     metagpt "Write a cli snake game"
@@ -183,7 +183,7 @@ # ステップ 2: コンテナで metagpt デモを実行する
 # コンテナを起動し、その中でコマンドを実行することもできます
 docker run --name metagpt -d \
     --privileged \
-    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
+    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
     metagpt/metagpt:latest
 
@@ -194,7 +194,7 @@ # コンテナを起動し、その中でコマンドを実行することもで
 コマンド `docker run ...` は以下のことを行います:
 
 - 特権モードで実行し、ブラウザの実行権限を得る
-- ホスト設定ファイル `/opt/metagpt/config/key.yaml` をコンテナ `/app/metagpt/config/key.yaml` にマップします
+- ホスト設定ファイル `/opt/metagpt/config/config2.yaml` をコンテナ `/app/metagpt/config/config2.yaml` にマップします
 - ホストディレクトリ `/opt/metagpt/workspace` をコンテナディレクトリ `/app/metagpt/workspace` にマップするs
 - デモコマンド `metagpt "Write a cli snake game"` を実行する
 
@@ -208,19 +208,14 @@ # また、自分で metagpt イメージを構築することもできます。
 
 ## 設定
 
-- `OPENAI_API_KEY` を `config/key.yaml / config/config.yaml / env` のいずれかで設定します。
-- 優先順位は: `config/key.yaml > config/config.yaml > env` の順です。
+- `api_key` を `~/.metagpt/config2.yaml / config/config2.yaml` のいずれかで設定します。
+- 優先順位は: `~/.metagpt/config2.yaml > config/config2.yaml > env` の順です。
 
 ```bash
 # 設定ファイルをコピーし、必要な修正を加える。
-cp config/config.yaml config/key.yaml
+cp config/config2.yaml ~/.metagpt/config2.yaml
 ```
 
-| 変数名                                  | config/key.yaml                           | env                                             |
-| --------------------------------------- | ----------------------------------------- | ----------------------------------------------- |
-| OPENAI_API_KEY # 自分のキーに置き換える | OPENAI_API_KEY: "sk-..."                  | export OPENAI_API_KEY="sk-..."                  |
-| OPENAI_BASE_URL # オプション            | OPENAI_BASE_URL: "https://<YOUR_SITE>/v1" | export OPENAI_BASE_URL="https://<YOUR_SITE>/v1" |
-
 ## チュートリアル: スタートアップの開始
 
 ```shell

docs/ROADMAP.md

@@ -76,9 +76,8 @@ ### Tasks
    2. ~~Support Azure asynchronous API~~
    3. Support streaming version of all APIs
    4. ~~Make gpt-3.5-turbo available (HARD)~~
-   5. Support 
 10. Other
     1. ~~Clean up existing unused code~~
-    2. Unify all code styles and establish contribution standards
+    2. ~~Unify all code styles and establish contribution standards~~
     3. ~~Multi-language support~~
     4. ~~Multi-programming-language support~~

docs/install/cli_install.md

@@ -9,17 +9,29 @@ ### Support System and version
 
 ### Detail Installation
 ```bash
-# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.)
-npm --version
-sudo npm install -g @mermaid-js/mermaid-cli
-
-# Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using:
+# Step 1: Ensure that Python 3.9+ is installed on your system. You can check this by using:
+# You can use conda to initialize a new python env
+#     conda create -n metagpt python=3.9
+#     conda activate metagpt
 python3 --version
 
-# Step 3: Clone the repository to your local machine, and install it.
+# Step 2: Clone the repository to your local machine for latest version, and install it.
 git clone https://github.com/geekan/MetaGPT.git
 cd MetaGPT
-pip install -e.
+pip3 install -e .     # or pip3 install metagpt  # for stable version
+
+# Step 3: setup your LLM key in the config2.yaml file
+mkdir ~/.metagpt
+cp config/config2.yaml ~/.metagpt/config2.yaml
+vim ~/.metagpt/config2.yaml
+
+# Step 4: run metagpt cli
+metagpt "Create a 2048 game in python"
+
+# Step 5 [Optional]: If you want to save the artifacts like diagrams such as quadrant chart, system designs, sequence flow in the workspace, you can execute the step before Step 3. By default, the framework is compatible, and the entire process can be run completely without executing this step.
+# If executing, ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js official website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.)
+npm --version
+sudo npm install -g @mermaid-js/mermaid-cli
 ```
 
 **Note:**
@@ -33,11 +45,12 @@ # Step 3: Clone the repository to your local machine, and install it.
   npm install @mermaid-js/mermaid-cli
   ```
 
-- don't forget to the configuration for mmdc in config.yml
+- don't forget to the configuration for mmdc path in config.yml
 
-  ```yml
-  PUPPETEER_CONFIG: "./config/puppeteer-config.json"
-  MMDC: "./node_modules/.bin/mmdc"
+  ```yaml
+  mermaid:
+    puppeteer_config: "./config/puppeteer-config.json"
+    path: "./node_modules/.bin/mmdc"
   ```
 
 - if `pip install -e.` fails with error `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`, try instead running `pip install -e. --user`
@@ -59,12 +72,13 @@ # Step 3: Clone the repository to your local machine, and install it.
     playwright install --with-deps chromium
     ```
 
-    - **modify `config.yaml`**
+    - **modify `config2.yaml`**
 
-    uncomment MERMAID_ENGINE from config.yaml and change it to `playwright`
+    change mermaid.engine to `playwright`
 
     ```yaml
-    MERMAID_ENGINE: playwright
+    mermaid:
+      engine: playwright
     ```
 
   - pyppeteer
@@ -88,22 +102,24 @@ # Step 3: Clone the repository to your local machine, and install it.
     pyppeteer-install
     ```
 
-    - **modify `config.yaml`**
+    - **modify `config2.yaml`**
 
-    uncomment MERMAID_ENGINE from config.yaml and change it to `pyppeteer`
+    change mermaid.engine to `pyppeteer`
 
     ```yaml
-    MERMAID_ENGINE: pyppeteer
+    mermaid:
+      engine: pyppeteer
     ```
 
   - mermaid.ink
-    - **modify `config.yaml`**
-
-    uncomment MERMAID_ENGINE from config.yaml and change it to `ink`
+    - **modify `config2.yaml`**
+    
+    change mermaid.engine to `ink`
 
     ```yaml
-    MERMAID_ENGINE: ink
+    mermaid:
+      engine: ink
     ```  
 
     Note: this method does not support pdf export.
-    
+    

docs/install/cli_install_cn.md

@@ -10,17 +10,29 @@ ### 支持的系统和版本
 ### 详细安装
 
 ```bash
-# 第 1 步：确保您的系统上安装了 NPM。并使用npm安装mermaid-js
-npm --version
-sudo npm install -g @mermaid-js/mermaid-cli
-
-# 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下命令进行检查：
+# 步骤 1: 确保您的系统安装了 Python 3.9 或更高版本。您可以使用以下命令来检查:
+# 您可以使用 conda 来初始化一个新的 Python 环境
+#     conda create -n metagpt python=3.9
+#     conda activate metagpt
 python3 --version
 
-# 第 3 步：克隆仓库到您的本地机器，并进行安装。
+# 步骤 2: 克隆仓库到您的本地机器以获取最新版本，并安装它。
 git clone https://github.com/geekan/MetaGPT.git
 cd MetaGPT
-pip install -e.
+pip3 install -e .     # 或 pip3 install metagpt  # 用于稳定版本
+
+# 步骤 3: 在 config2.yaml 文件中设置您的 LLM 密钥
+mkdir ~/.metagpt
+cp config/config2.yaml ~/.metagpt/config2.yaml
+vim ~/.metagpt/config2.yaml
+
+# 步骤 4: 运行 metagpt 命令行界面
+metagpt "用 python 创建一个 2048 游戏"
+
+# 步骤 5 [可选]: 如果您想保存诸如象限图、系统设计、序列流等图表作为工作空间的工件，您可以在执行步骤 3 之前执行此步骤。默认情况下，该框架是兼容的，整个过程可以完全不执行此步骤而运行。
+# 如果执行此步骤，请确保您的系统上安装了 NPM。然后安装 mermaid-js。（如果您的计算机中没有 npm，请访问 Node.js 官方网站 https://nodejs.org/ 安装 Node.js，然后您将在计算机中拥有 npm 工具。）
+npm --version
+sudo npm install -g @mermaid-js/mermaid-cli
 ```
 
 **注意：**
@@ -33,11 +45,12 @@ # 第 3 步：克隆仓库到您的本地机器，并进行安装。
     npm install @mermaid-js/mermaid-cli
     ```
 
-- 不要忘记在config.yml中为mmdc配置配置，
+- 不要忘记在config.yml中为mmdc配置
 
     ```yml
-    PUPPETEER_CONFIG: "./config/puppeteer-config.json"
-    MMDC: "./node_modules/.bin/mmdc"
+    mermaid:
+      puppeteer_config: "./config/puppeteer-config.json"
+      path: "./node_modules/.bin/mmdc"
     ```
 
 - 如果`pip install -e.`失败并显示错误`[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`，请尝试使用`pip install -e. --user`运行。

docs/install/docker_install.md

@@ -3,16 +3,16 @@ ## Docker Installation
 ### Use default MetaGPT image
 
 ```bash
-# Step 1: Download metagpt official image and prepare config.yaml
+# Step 1: Download metagpt official image and prepare config2.yaml
 docker pull metagpt/metagpt:latest
 mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml
-vim /opt/metagpt/config/key.yaml # Change the config
+docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
+vim /opt/metagpt/config/config2.yaml # Change the config
 
 # Step 2: Run metagpt demo with container
 docker run --rm \
     --privileged \
-    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
+    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
     metagpt/metagpt:latest \
     metagpt "Write a cli snake game"
@@ -20,7 +20,7 @@ # Step 2: Run metagpt demo with container
 # You can also start a container and execute commands in it
 docker run --name metagpt -d \
     --privileged \
-    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
+    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
     metagpt/metagpt:latest
 
@@ -31,7 +31,7 @@ # You can also start a container and execute commands in it
 The command `docker run ...` do the following things:
 
 - Run in privileged mode to have permission to run the browser
-- Map host configure file `/opt/metagpt/config/key.yaml` to container `/app/metagpt/config/key.yaml`
+- Map host configure file `/opt/metagpt/config/config2.yaml` to container `/app/metagpt/config/config2.yaml`
 - Map host directory `/opt/metagpt/workspace` to container `/app/metagpt/workspace`
 - Execute the demo command `metagpt "Write a cli snake game"`
 

docs/install/docker_install_cn.md

@@ -3,16 +3,16 @@ ## Docker安装
 ### 使用MetaGPT镜像
 
 ```bash
-# 步骤1: 下载metagpt官方镜像并准备好config.yaml
+# 步骤1: 下载metagpt官方镜像并准备好config2.yaml
 docker pull metagpt/metagpt:latest
 mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config.yaml > /opt/metagpt/config/key.yaml
-vim /opt/metagpt/config/key.yaml # 修改配置文件
+docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
+vim /opt/metagpt/config/config2.yaml # 修改配置文件
 
 # 步骤2: 使用容器运行metagpt演示
 docker run --rm \
     --privileged \
-    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
+    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
     metagpt/metagpt:latest \
     metagpt "Write a cli snake game"
@@ -20,7 +20,7 @@ # 步骤2: 使用容器运行metagpt演示
 # 您也可以启动一个容器并在其中执行命令
 docker run --name metagpt -d \
     --privileged \
-    -v /opt/metagpt/config/key.yaml:/app/metagpt/config/key.yaml \
+    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
     -v /opt/metagpt/workspace:/app/metagpt/workspace \
     metagpt/metagpt:latest
 
@@ -31,7 +31,7 @@ # 您也可以启动一个容器并在其中执行命令
 `docker run ...`做了以下事情:
 
 - 以特权模式运行，有权限运行浏览器
-- 将主机文件 `/opt/metagpt/config/key.yaml` 映射到容器文件 `/app/metagpt/config/key.yaml`
+- 将主机文件 `/opt/metagpt/config/config2.yaml` 映射到容器文件 `/app/metagpt/config/config2.yaml`
 - 将主机目录 `/opt/metagpt/workspace` 映射到容器目录 `/app/metagpt/workspace`
 - 执行示例命令 `metagpt "Write a cli snake game"`
 

docs/tutorial/usage.md

@@ -2,19 +2,14 @@ ## MetaGPT Usage
 
 ### Configuration
 
-- Configure your `OPENAI_API_KEY` in any of `config/key.yaml / config/config.yaml / env`
-- Priority order: `config/key.yaml > config/config.yaml > env`
+- Configure your `api_key` in any of `~/.metagpt/config2.yaml / config/config2.yaml`
+- Priority order: `~/.metagpt/config2.yaml > config/config2.yaml`
 
 ```bash
 # Copy the configuration file and make the necessary modifications.
-cp config/config.yaml config/key.yaml
+cp config/config2.yaml ~/.metagpt/config2.yaml
 ```
 
-| Variable Name                              | config/key.yaml                           | env                                             |
-| ------------------------------------------ | ----------------------------------------- | ----------------------------------------------- |
-| OPENAI_API_KEY # Replace with your own key | OPENAI_API_KEY: "sk-..."                  | export OPENAI_API_KEY="sk-..."                  |
-| OPENAI_BASE_URL # Optional                 | OPENAI_BASE_URL: "https://<YOUR_SITE>/v1" | export OPENAI_BASE_URL="https://<YOUR_SITE>/v1" |
-
 ### Initiating a startup
 
 ```shell
@@ -39,29 +34,28 @@ ### Preference of Platform or Tool
 ### Usage
 
 ```
-NAME
-    metagpt - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities.
-
-SYNOPSIS
-    metagpt IDEA <flags>
-
-DESCRIPTION
-    We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities.
-
-POSITIONAL ARGUMENTS
-    IDEA
-        Type: str
-        Your innovative idea, such as "Creating a snake game."
-
-FLAGS
-    --investment=INVESTMENT
-        Type: float
-        Default: 3.0
-        As an investor, you have the opportunity to contribute a certain dollar amount to this AI company.
-    --n_round=N_ROUND
-        Type: int
-        Default: 5
-
-NOTES
-    You can also use flags syntax for POSITIONAL ARGUMENTS
+ Usage: metagpt [OPTIONS] [IDEA]                                                                                                                                                                                          
+                                                                                                                                                                                                                          
+ Start a new project.                                                                                                                                                                                                     
+                                                                                                                                                                                                                          
+╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+│   idea      [IDEA]  Your innovative idea, such as 'Create a 2048 game.' [default: None]                                                                                                                                │
+╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+│ --investment                                     FLOAT    Dollar amount to invest in the AI company. [default: 3.0]                                                                                                    │
+│ --n-round                                        INTEGER  Number of rounds for the simulation. [default: 5]                                                                                                            │
+│ --code-review                --no-code-review             Whether to use code review. [default: code-review]                                                                                                           │
+│ --run-tests                  --no-run-tests               Whether to enable QA for adding & running tests. [default: no-run-tests]                                                                                     │
+│ --implement                  --no-implement               Enable or disable code implementation. [default: implement]                                                                                                  │
+│ --project-name                                   TEXT     Unique project name, such as 'game_2048'.                                                                                                                    │
+│ --inc                        --no-inc                     Incremental mode. Use it to coop with existing repo. [default: no-inc]                                                                                       │
+│ --project-path                                   TEXT     Specify the directory path of the old version project to fulfill the incremental requirements.                                                               │
+│ --reqa-file                                      TEXT     Specify the source file name for rewriting the quality assurance code.                                                                                       │
+│ --max-auto-summarize-code                        INTEGER  The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the      │
+│                                                           workflow.                                                                                                                                                    │
+│                                                           [default: 0]                                                                                                                                                 │
+│ --recover-path                                   TEXT     recover the project from existing serialized storage [default: None]                                                                                         │
+│ --init-config                --no-init-config             Initialize the configuration file for MetaGPT. [default: no-init-config]                                                                                     │
+│ --help                                                    Show this message and exit.                                                                                                                                  │
+╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```

docs/tutorial/usage_cn.md

@@ -2,19 +2,14 @@ ## MetaGPT 使用
 
 ### 配置
 
-- 在 `config/key.yaml / config/config.yaml / env` 中配置您的 `OPENAI_API_KEY`
-- 优先级顺序：`config/key.yaml > config/config.yaml > env`
+- 在 `~/.metagpt/config2.yaml / config/config2.yaml` 中配置您的 `api_key`
+- 优先级顺序：`~/.metagpt/config2.yaml > config/config2.yaml`
 
 ```bash
 # 复制配置文件并进行必要的修改
-cp config/config.yaml config/key.yaml
+cp config/config2.yaml ~/.metagpt/config2.yaml
 ```
 
-| 变量名                              | config/key.yaml                           | env                                             |
-| ----------------------------------- | ----------------------------------------- | ----------------------------------------------- |
-| OPENAI_API_KEY # 用您自己的密钥替换 | OPENAI_API_KEY: "sk-..."                  | export OPENAI_API_KEY="sk-..."                  |
-| OPENAI_BASE_URL # 可选              | OPENAI_BASE_URL: "https://<YOUR_SITE>/v1" | export OPENAI_BASE_URL="https://<YOUR_SITE>/v1" |
-
 ### 示例：启动一个创业公司
 
 ```shell
@@ -35,29 +30,28 @@ ### 平台或工具的倾向性
 ### 使用
 
 ```
-名称
-    metagpt - 我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。
-
-概要
-    metagpt IDEA <flags>
-
-描述
-    我们是一家AI软件创业公司。通过投资我们，您将赋能一个充满无限可能的未来。
-
-位置参数
-    IDEA
-        类型: str
-        您的创新想法，例如"写一个命令行贪吃蛇。"
-
-标志
-    --investment=INVESTMENT
-        类型: float
-        默认值: 3.0
-        作为投资者，您有机会向这家AI公司投入一定的美元金额。
-    --n_round=N_ROUND
-        类型: int
-        默认值: 5
-
-备注
-    您也可以用`标志`的语法，来处理`位置参数`
+ Usage: metagpt [OPTIONS] [IDEA]                                                                                                                                                                                          
+                                                                                                                                                                                                                          
+ Start a new project.                                                                                                                                                                                                     
+                                                                                                                                                                                                                          
+╭─ Arguments ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+│   idea      [IDEA]  Your innovative idea, such as 'Create a 2048 game.' [default: None]                                                                                                                                │
+╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
+│ --investment                                     FLOAT    Dollar amount to invest in the AI company. [default: 3.0]                                                                                                    │
+│ --n-round                                        INTEGER  Number of rounds for the simulation. [default: 5]                                                                                                            │
+│ --code-review                --no-code-review             Whether to use code review. [default: code-review]                                                                                                           │
+│ --run-tests                  --no-run-tests               Whether to enable QA for adding & running tests. [default: no-run-tests]                                                                                     │
+│ --implement                  --no-implement               Enable or disable code implementation. [default: implement]                                                                                                  │
+│ --project-name                                   TEXT     Unique project name, such as 'game_2048'.                                                                                                                    │
+│ --inc                        --no-inc                     Incremental mode. Use it to coop with existing repo. [default: no-inc]                                                                                       │
+│ --project-path                                   TEXT     Specify the directory path of the old version project to fulfill the incremental requirements.                                                               │
+│ --reqa-file                                      TEXT     Specify the source file name for rewriting the quality assurance code.                                                                                       │
+│ --max-auto-summarize-code                        INTEGER  The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the      │
+│                                                           workflow.                                                                                                                                                    │
+│                                                           [default: 0]                                                                                                                                                 │
+│ --recover-path                                   TEXT     recover the project from existing serialized storage [default: None]                                                                                         │
+│ --init-config                --no-init-config             Initialize the configuration file for MetaGPT. [default: no-init-config]                                                                                     │
+│ --help                                                    Show this message and exit.                                                                                                                                  │
+╰────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```

examples/crawl_webpage.py

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+"""
+@Date    :   2024/01/24 15:11:27
+@Author  :   orange-crow
+@File    :   crawl_webpage.py
+"""
+
+from metagpt.roles.ci.code_interpreter import CodeInterpreter
+
+
+async def main():
+    prompt = """Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/,
+    and save it to a csv file. paper title must include `multiagent` or `large language model`. *notice: print key variables*"""
+    ci = CodeInterpreter(goal=prompt, use_tools=True)
+
+    await ci.run(prompt)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

examples/dalle_gpt4v_agent.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : use gpt4v to improve prompt and draw image with dall-e-3
+
+"""set `model: "gpt-4-vision-preview"` in `config2.yaml` first"""
+
+import asyncio
+
+from PIL import Image
+
+from metagpt.actions.action import Action
+from metagpt.logs import logger
+from metagpt.roles.role import Role
+from metagpt.schema import Message
+from metagpt.utils.common import encode_image
+
+
+class GenAndImproveImageAction(Action):
+    save_image: bool = True
+
+    async def generate_image(self, prompt: str) -> Image:
+        imgs = await self.llm.gen_image(model="dall-e-3", prompt=prompt)
+        return imgs[0]
+
+    async def refine_prompt(self, old_prompt: str, image: Image) -> str:
+        msg = (
+            f"You are a creative painter, with the given generated image and old prompt: {old_prompt}, "
+            f"please refine the prompt and generate new one. Just output the new prompt."
+        )
+        b64_img = encode_image(image)
+        new_prompt = await self.llm.aask(msg=msg, images=[b64_img])
+        return new_prompt
+
+    async def evaluate_images(self, old_prompt: str, images: list[Image]) -> str:
+        msg = (
+            "With the prompt and two generated image, to judge if the second one is better than the first one. "
+            "If so, just output True else output False"
+        )
+        b64_imgs = [encode_image(img) for img in images]
+        res = await self.llm.aask(msg=msg, images=b64_imgs)
+        return res
+
+    async def run(self, messages: list[Message]) -> str:
+        prompt = messages[-1].content
+
+        old_img: Image = await self.generate_image(prompt)
+        new_prompt = await self.refine_prompt(old_prompt=prompt, image=old_img)
+        logger.info(f"original prompt: {prompt}")
+        logger.info(f"refined prompt: {new_prompt}")
+        new_img: Image = await self.generate_image(new_prompt)
+        if self.save_image:
+            old_img.save("./img_by-dall-e_old.png")
+            new_img.save("./img_by-dall-e_new.png")
+        res = await self.evaluate_images(old_prompt=prompt, images=[old_img, new_img])
+        opinion = f"The second generated image is better than the first one: {res}"
+        logger.info(f"evaluate opinion: {opinion}")
+        return opinion
+
+
+class Painter(Role):
+    name: str = "MaLiang"
+    profile: str = "Painter"
+    goal: str = "to generate fine painting"
+
+    def __init__(self, **data):
+        super().__init__(**data)
+
+        self.set_actions([GenAndImproveImageAction])
+
+
+async def main():
+    role = Painter()
+    await role.run(with_message="a girl with flowers")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/imitate_webpage.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/01/15
+@Author  : mannaandpoem
+@File    : imitate_webpage.py
+"""
+from metagpt.roles.ci.code_interpreter import CodeInterpreter
+
+
+async def main():
+    web_url = "https://pytorch.org/"
+    prompt = f"""This is a URL of webpage: '{web_url}' .
+Firstly, utilize Selenium and WebDriver for rendering. 
+Secondly, convert image to a webpage including HTML, CSS and JS in one go. 
+Finally, save webpage in a text file. 
+Note: All required dependencies and environments have been fully installed and configured."""
+    ci = CodeInterpreter(goal=prompt, use_tools=True)
+
+    await ci.run(prompt)
+
+
+if __name__ == "__main__":
+    import asyncio
+
+    asyncio.run(main())

examples/llm_hello_world.py

@@ -23,6 +23,10 @@ async def main():
     # streaming mode, much slower
     await llm.acompletion_text(hello_msg, stream=True)
 
+    # check completion if exist to test llm complete functions
+    if hasattr(llm, "completion"):
+        logger.info(llm.completion(hello_msg))
+
 
 if __name__ == "__main__":
     asyncio.run(main())

examples/llm_vision.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : example to run the ability of LLM vision
+
+import asyncio
+from pathlib import Path
+
+from metagpt.llm import LLM
+from metagpt.utils.common import encode_image
+
+
+async def main():
+    llm = LLM()
+
+    # check if the configured llm supports llm-vision capacity. If not, it will throw a error
+    invoice_path = Path(__file__).parent.joinpath("..", "tests", "data", "invoices", "invoice-2.png")
+    img_base64 = encode_image(invoice_path)
+    res = await llm.aask(msg="if this is a invoice, just return True else return False", images=[img_base64])
+    assert "true" in res.lower()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

examples/sd_tool_usage.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# @Date    : 1/11/2024 7:06 PM
+# @Author  : stellahong (stellahong@fuzhi.ai)
+# @Desc    :
+import asyncio
+
+from metagpt.roles.ci.code_interpreter import CodeInterpreter
+
+
+async def main(requirement: str = ""):
+    code_interpreter = CodeInterpreter(use_tools=True, goal=requirement)
+    await code_interpreter.run(requirement)
+
+
+if __name__ == "__main__":
+    sd_url = "http://your.sd.service.ip:port"
+    requirement = (
+        f"I want to generate an image of a beautiful girl using the stable diffusion text2image tool, sd_url={sd_url}"
+    )
+
+    asyncio.run(main(requirement))

examples/search_with_specific_engine.py

@@ -5,17 +5,20 @@
 import asyncio
 
 from metagpt.roles import Searcher
-from metagpt.tools import SearchEngineType
+from metagpt.tools.search_engine import SearchEngine, SearchEngineType
 
 
 async def main():
     question = "What are the most interesting human facts?"
+    kwargs = {"api_key": "", "cse_id": "", "proxy": None}
     # Serper API
-    # await Searcher(engine=SearchEngineType.SERPER_GOOGLE).run(question)
+    # await Searcher(search_engine=SearchEngine(engine=SearchEngineType.SERPER_GOOGLE, **kwargs)).run(question)
     # SerpAPI
-    await Searcher(engine=SearchEngineType.SERPAPI_GOOGLE).run(question)
+    # await Searcher(search_engine=SearchEngine(engine=SearchEngineType.SERPAPI_GOOGLE, **kwargs)).run(question)
     # Google API
-    # await Searcher(engine=SearchEngineType.DIRECT_GOOGLE).run(question)
+    # await Searcher(search_engine=SearchEngine(engine=SearchEngineType.DIRECT_GOOGLE, **kwargs)).run(question)
+    # DDG API
+    await Searcher(search_engine=SearchEngine(engine=SearchEngineType.DUCK_DUCK_GO, **kwargs)).run(question)
 
 
 if __name__ == "__main__":

examples/write_novel.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/2/1 12:01
+@Author  : alexanderwu
+@File    : write_novel.py
+"""
+import asyncio
+from typing import List
+
+from pydantic import BaseModel, Field
+
+from metagpt.actions.action_node import ActionNode
+from metagpt.llm import LLM
+
+
+class Novel(BaseModel):
+    name: str = Field(default="The Lord of the Rings", description="The name of the novel.")
+    user_group: str = Field(default="...", description="The user group of the novel.")
+    outlines: List[str] = Field(
+        default=["Chapter 1: ...", "Chapter 2: ...", "Chapter 3: ..."],
+        description="The outlines of the novel. No more than 10 chapters.",
+    )
+    background: str = Field(default="...", description="The background of the novel.")
+    character_names: List[str] = Field(default=["Frodo", "Gandalf", "Sauron"], description="The characters.")
+    conflict: str = Field(default="...", description="The conflict of the characters.")
+    plot: str = Field(default="...", description="The plot of the novel.")
+    ending: str = Field(default="...", description="The ending of the novel.")
+
+
+class Chapter(BaseModel):
+    name: str = Field(default="Chapter 1", description="The name of the chapter.")
+    content: str = Field(default="...", description="The content of the chapter. No more than 1000 words.")
+
+
+async def generate_novel():
+    instruction = (
+        "Write a novel named 'Harry Potter in The Lord of the Rings'. "
+        "Fill the empty nodes with your own ideas. Be creative! Use your own words!"
+        "I will tip you $100,000 if you write a good novel."
+    )
+    novel_node = await ActionNode.from_pydantic(Novel).fill(context=instruction, llm=LLM())
+    chap_node = await ActionNode.from_pydantic(Chapter).fill(
+        context=f"### instruction\n{instruction}\n### novel\n{novel_node.content}", llm=LLM()
+    )
+    print(chap_node.content)
+
+
+asyncio.run(generate_novel())

metagpt/actions/__init__.py

@@ -22,6 +22,9 @@ from metagpt.actions.write_code_review import WriteCodeReview
 from metagpt.actions.write_prd import WritePRD
 from metagpt.actions.write_prd_review import WritePRDReview
 from metagpt.actions.write_test import WriteTest
+from metagpt.actions.ci.execute_nb_code import ExecuteNbCode
+from metagpt.actions.ci.write_analysis_code import WriteCodeWithoutTools, WriteCodeWithTools
+from metagpt.actions.ci.write_plan import WritePlan
 
 
 class ActionType(Enum):
@@ -42,6 +45,10 @@ class ActionType(Enum):
     COLLECT_LINKS = CollectLinks
     WEB_BROWSE_AND_SUMMARIZE = WebBrowseAndSummarize
     CONDUCT_RESEARCH = ConductResearch
+    EXECUTE_NB_CODE = ExecuteNbCode
+    WRITE_CODE_WITHOUT_TOOLS = WriteCodeWithoutTools
+    WRITE_CODE_WITH_TOOLS = WriteCodeWithTools
+    WRITE_PLAN = WritePlan
 
 
 __all__ = [

metagpt/actions/action.py

@@ -15,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
 from metagpt.actions.action_node import ActionNode
 from metagpt.context_mixin import ContextMixin
 from metagpt.schema import (
+    CodePlanAndChangeContext,
     CodeSummarizeContext,
     CodingContext,
     RunCodeContext,
@@ -28,14 +29,18 @@ class Action(SerializationMixin, ContextMixin, BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     name: str = ""
-    i_context: Union[dict, CodingContext, CodeSummarizeContext, TestingContext, RunCodeContext, str, None] = ""
+    i_context: Union[
+        dict, CodingContext, CodeSummarizeContext, TestingContext, RunCodeContext, CodePlanAndChangeContext, str, None
+    ] = ""
     prefix: str = ""  # aask*时会加上prefix，作为system_message
     desc: str = ""  # for skill manager
     node: ActionNode = Field(default=None, exclude=True)
 
     @property
-    def project_repo(self):
-        return ProjectRepo(self.context.git_repo)
+    def repo(self) -> ProjectRepo:
+        if not self.context.repo:
+            self.context.repo = ProjectRepo(self.context.git_repo)
+        return self.context.repo
 
     @property
     def prompt_schema(self):

metagpt/actions/action_graph.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2024/1/30 13:52
+@Author  : alexanderwu
+@File    : action_graph.py
+"""
+from __future__ import annotations
+
+# from metagpt.actions.action_node import ActionNode
+
+
+class ActionGraph:
+    """ActionGraph: a directed graph to represent the dependency between actions."""
+
+    def __init__(self):
+        self.nodes = {}
+        self.edges = {}
+        self.execution_order = []
+
+    def add_node(self, node):
+        """Add a node to the graph"""
+        self.nodes[node.key] = node
+
+    def add_edge(self, from_node: "ActionNode", to_node: "ActionNode"):
+        """Add an edge to the graph"""
+        if from_node.key not in self.edges:
+            self.edges[from_node.key] = []
+        self.edges[from_node.key].append(to_node.key)
+        from_node.add_next(to_node)
+        to_node.add_prev(from_node)
+
+    def topological_sort(self):
+        """Topological sort the graph"""
+        visited = set()
+        stack = []
+
+        def visit(k):
+            if k not in visited:
+                visited.add(k)
+                if k in self.edges:
+                    for next_node in self.edges[k]:
+                        visit(next_node)
+                stack.insert(0, k)
+
+        for key in self.nodes:
+            visit(key)
+
+        self.execution_order = stack

metagpt/actions/action_node.py

@@ -9,10 +9,11 @@ NOTE: You should use typing.List instead of list to do type annotation. Because
   we can use typing to extract the type of the node, but we cannot use built-in list to extract.
 """
 import json
+import typing
 from enum import Enum
 from typing import Any, Dict, List, Optional, Tuple, Type, Union
 
-from pydantic import BaseModel, create_model, model_validator
+from pydantic import BaseModel, Field, create_model, model_validator
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions.action_outcls_registry import register_action_outcls
@@ -39,7 +40,6 @@ TAG = "CONTENT"
 LANGUAGE_CONSTRAINT = "Language: Please use the same language as Human INPUT."
 FORMAT_CONSTRAINT = f"Format: output wrapped inside [{TAG}][/{TAG}] like format example, nothing else."
 
-
 SIMPLE_TEMPLATE = """
 ## context
 {context}
@@ -61,7 +61,7 @@ Follow instructions of nodes, generate output and make sure it follows the forma
 
 REVIEW_TEMPLATE = """
 ## context
-Compare the keys of nodes_output and the corresponding requirements one by one. If a key that does not match the requirement is found, provide the comment content on how to modify it. No output is required for matching keys.
+Compare the key's value of nodes_output and the corresponding requirements one by one. If a key's value that does not match the requirement is found, provide the comment content on how to modify it. No output is required for matching keys.
 
 ### nodes_output
 {nodes_output}
@@ -86,7 +86,7 @@ Compare the keys of nodes_output and the corresponding requirements one by one.
 {constraint}
 
 ## action
-generate output and make sure it follows the format example.
+Follow format example's {prompt_schema} format, generate output and make sure it follows the format example.
 """
 
 REVISE_TEMPLATE = """
@@ -108,7 +108,7 @@ change the nodes_output key's value to meet its comment and no need to add extra
 {constraint}
 
 ## action
-generate output and make sure it follows the format example.
+Follow format example's {prompt_schema} format, generate output and make sure it follows the format example.
 """
 
 
@@ -131,6 +131,8 @@ class ActionNode:
 
     # Action Input
     key: str  # Product Requirement / File list / Code
+    func: typing.Callable  # 与节点相关联的函数或LLM调用
+    params: Dict[str, Type]  # 输入参数的字典，键为参数名，值为参数类型
     expected_type: Type  # such as str / int / float etc.
     # context: str  # everything in the history.
     instruction: str  # the instructions should be followed.
@@ -140,6 +142,10 @@ class ActionNode:
     content: str
     instruct_content: BaseModel
 
+    # For ActionGraph
+    prevs: List["ActionNode"]  # previous nodes
+    nexts: List["ActionNode"]  # next nodes
+
     def __init__(
         self,
         key: str,
@@ -157,6 +163,8 @@ class ActionNode:
         self.content = content
         self.children = children if children is not None else {}
         self.schema = schema
+        self.prevs = []
+        self.nexts = []
 
     def __str__(self):
         return (
@@ -167,6 +175,14 @@ class ActionNode:
     def __repr__(self):
         return self.__str__()
 
+    def add_prev(self, node: "ActionNode"):
+        """增加前置ActionNode"""
+        self.prevs.append(node)
+
+    def add_next(self, node: "ActionNode"):
+        """增加后置ActionNode"""
+        self.nexts.append(node)
+
     def add_child(self, node: "ActionNode"):
         """增加子ActionNode"""
         self.children[node.key] = node
@@ -186,25 +202,38 @@ class ActionNode:
         obj.add_children(nodes)
         return obj
 
-    def get_children_mapping(self, exclude=None) -> Dict[str, Tuple[Type, Any]]:
-        """获得子ActionNode的字典，以key索引"""
+    def _get_children_mapping(self, exclude=None) -> Dict[str, Any]:
+        """获得子ActionNode的字典，以key索引，支持多级结构。"""
         exclude = exclude or []
-        return {k: (v.expected_type, ...) for k, v in self.children.items() if k not in exclude}
 
-    def get_self_mapping(self) -> Dict[str, Tuple[Type, Any]]:
+        def _get_mapping(node: "ActionNode") -> Dict[str, Any]:
+            mapping = {}
+            for key, child in node.children.items():
+                if key in exclude:
+                    continue
+                # 对于嵌套的子节点，递归调用 _get_mapping
+                if child.children:
+                    mapping[key] = _get_mapping(child)
+                else:
+                    mapping[key] = (child.expected_type, Field(default=child.example, description=child.instruction))
+            return mapping
+
+        return _get_mapping(self)
+
+    def _get_self_mapping(self) -> Dict[str, Tuple[Type, Any]]:
         """get self key: type mapping"""
         return {self.key: (self.expected_type, ...)}
 
     def get_mapping(self, mode="children", exclude=None) -> Dict[str, Tuple[Type, Any]]:
         """get key: type mapping under mode"""
         if mode == "children" or (mode == "auto" and self.children):
-            return self.get_children_mapping(exclude=exclude)
-        return {} if exclude and self.key in exclude else self.get_self_mapping()
+            return self._get_children_mapping(exclude=exclude)
+        return {} if exclude and self.key in exclude else self._get_self_mapping()
 
     @classmethod
     @register_action_outcls
     def create_model_class(cls, class_name: str, mapping: Dict[str, Tuple[Type, Any]]):
-        """基于pydantic v1的模型动态生成，用来检验结果类型正确性"""
+        """基于pydantic v2的模型动态生成，用来检验结果类型正确性"""
 
         def check_fields(cls, values):
             required_fields = set(mapping.keys())
@@ -219,7 +248,17 @@ class ActionNode:
 
         validators = {"check_missing_fields_validator": model_validator(mode="before")(check_fields)}
 
-        new_class = create_model(class_name, __validators__=validators, **mapping)
+        new_fields = {}
+        for field_name, field_value in mapping.items():
+            if isinstance(field_value, dict):
+                # 对于嵌套结构，递归创建模型类
+                nested_class_name = f"{class_name}_{field_name}"
+                nested_class = cls.create_model_class(nested_class_name, field_value)
+                new_fields[field_name] = (nested_class, ...)
+            else:
+                new_fields[field_name] = field_value
+
+        new_class = create_model(class_name, __validators__=validators, **new_fields)
         return new_class
 
     def create_class(self, mode: str = "auto", class_name: str = None, exclude=None):
@@ -227,39 +266,48 @@ class ActionNode:
         mapping = self.get_mapping(mode=mode, exclude=exclude)
         return self.create_model_class(class_name, mapping)
 
-    def create_children_class(self, exclude=None):
+    def _create_children_class(self, exclude=None):
         """使用object内有的字段直接生成model_class"""
         class_name = f"{self.key}_AN"
-        mapping = self.get_children_mapping(exclude=exclude)
+        mapping = self._get_children_mapping(exclude=exclude)
         return self.create_model_class(class_name, mapping)
 
     def to_dict(self, format_func=None, mode="auto", exclude=None) -> Dict:
         """将当前节点与子节点都按照node: format的格式组织成字典"""
+        nodes = self._to_dict(format_func=format_func, mode=mode, exclude=exclude)
+        if not isinstance(nodes, dict):
+            nodes = {self.key: nodes}
+        return nodes
 
-        # 如果没有提供格式化函数，使用默认的格式化方式
+    def _to_dict(self, format_func=None, mode="auto", exclude=None) -> Dict:
+        """将当前节点与子节点都按照node: format的格式组织成字典"""
+
+        # 如果没有提供格式化函数，则使用默认的格式化函数
         if format_func is None:
-            format_func = lambda node: f"{node.instruction}"
+            format_func = lambda node: node.instruction
 
         # 使用提供的格式化函数来格式化当前节点的值
         formatted_value = format_func(self)
 
         # 创建当前节点的键值对
-        if mode == "children" or (mode == "auto" and self.children):
-            node_dict = {}
+        if (mode == "children" or mode == "auto") and self.children:
+            node_value = {}
         else:
-            node_dict = {self.key: formatted_value}
+            node_value = formatted_value
 
         if mode == "root":
-            return node_dict
+            return {self.key: node_value}
 
-        # 遍历子节点并递归调用 to_dict 方法
+        # 递归处理子节点
         exclude = exclude or []
-        for _, child_node in self.children.items():
-            if child_node.key in exclude:
+        for child_key, child_node in self.children.items():
+            if child_key in exclude:
                 continue
-            node_dict.update(child_node.to_dict(format_func))
+            # 递归调用 to_dict 方法并更新节点字典
+            child_dict = child_node._to_dict(format_func, mode, exclude)
+            node_value[child_key] = child_dict
 
-        return node_dict
+        return node_value
 
     def update_instruct_content(self, incre_data: dict[str, Any]):
         assert self.instruct_content
@@ -328,6 +376,17 @@ class ActionNode:
         if schema == "raw":
             return context + "\n\n## Actions\n" + LANGUAGE_CONSTRAINT + "\n" + self.instruction
 
+        ### 直接使用 pydantic BaseModel 生成 instruction 与 example，仅限 JSON
+        # child_class = self._create_children_class()
+        # node_schema = child_class.model_json_schema()
+        # defaults = {
+        #     k: str(v)
+        #     for k, v in child_class.model_fields.items()
+        #     if k not in exclude
+        # }
+        # instruction = node_schema
+        # example = json.dumps(defaults, indent=4)
+
         # FIXME: json instruction会带来格式问题，如："Project name": "web_2048  # 项目名称使用下划线",
         # compile example暂时不支持markdown
         instruction = self.compile_instruction(schema="markdown", mode=mode, exclude=exclude)
@@ -354,12 +413,13 @@ class ActionNode:
         prompt: str,
         output_class_name: str,
         output_data_mapping: dict,
+        images: Optional[Union[str, list[str]]] = None,
         system_msgs: Optional[list[str]] = None,
         schema="markdown",  # compatible to original format
         timeout=3,
     ) -> (str, BaseModel):
         """Use ActionOutput to wrap the output of aask"""
-        content = await self.llm.aask(prompt, system_msgs, timeout=timeout)
+        content = await self.llm.aask(prompt, system_msgs, images=images, timeout=timeout)
         logger.debug(f"llm raw output:\n{content}")
         output_class = self.create_model_class(output_class_name, output_data_mapping)
 
@@ -388,13 +448,15 @@ class ActionNode:
     def set_context(self, context):
         self.set_recursive("context", context)
 
-    async def simple_fill(self, schema, mode, timeout=3, exclude=None):
+    async def simple_fill(self, schema, mode, images: Optional[Union[str, list[str]]] = None, timeout=3, exclude=None):
         prompt = self.compile(context=self.context, schema=schema, mode=mode, exclude=exclude)
 
         if schema != "raw":
             mapping = self.get_mapping(mode, exclude=exclude)
             class_name = f"{self.key}_AN"
-            content, scontent = await self._aask_v1(prompt, class_name, mapping, schema=schema, timeout=timeout)
+            content, scontent = await self._aask_v1(
+                prompt, class_name, mapping, images=images, schema=schema, timeout=timeout
+            )
             self.content = content
             self.instruct_content = scontent
         else:
@@ -403,7 +465,17 @@ class ActionNode:
 
         return self
 
-    async def fill(self, context, llm, schema="json", mode="auto", strgy="simple", timeout=3, exclude=[]):
+    async def fill(
+        self,
+        context,
+        llm,
+        schema="json",
+        mode="auto",
+        strgy="simple",
+        images: Optional[Union[str, list[str]]] = None,
+        timeout=3,
+        exclude=[],
+    ):
         """Fill the node(s) with mode.
 
         :param context: Everything we should know when filling node.
@@ -419,6 +491,7 @@ class ActionNode:
         :param strgy: simple/complex
          - simple: run only once
          - complex: run each node
+        :param images: the list of image url or base64 for gpt4-v
         :param timeout: Timeout for llm invocation.
         :param exclude: The keys of ActionNode to exclude.
         :return: self
@@ -429,16 +502,16 @@ class ActionNode:
             schema = self.schema
 
         if strgy == "simple":
-            return await self.simple_fill(schema=schema, mode=mode, timeout=timeout, exclude=exclude)
+            return await self.simple_fill(schema=schema, mode=mode, images=images, timeout=timeout, exclude=exclude)
         elif strgy == "complex":
             # 这里隐式假设了拥有children
             tmp = {}
             for _, i in self.children.items():
                 if exclude and i.key in exclude:
                     continue
-                child = await i.simple_fill(schema=schema, mode=mode, timeout=timeout, exclude=exclude)
+                child = await i.simple_fill(schema=schema, mode=mode, images=images, timeout=timeout, exclude=exclude)
                 tmp.update(child.instruct_content.model_dump())
-            cls = self.create_children_class()
+            cls = self._create_children_class()
             self.instruct_content = cls(**tmp)
             return self
 
@@ -469,7 +542,10 @@ class ActionNode:
             return dict()
 
         prompt = template.format(
-            nodes_output=json.dumps(nodes_output, ensure_ascii=False, indent=4), tag=TAG, constraint=FORMAT_CONSTRAINT
+            nodes_output=json.dumps(nodes_output, ensure_ascii=False),
+            tag=TAG,
+            constraint=FORMAT_CONSTRAINT,
+            prompt_schema="json",
         )
 
         content = await self.llm.aask(prompt)
@@ -563,10 +639,11 @@ class ActionNode:
         instruction = self.compile_instruction(schema="markdown", mode="auto", exclude=exclude_keys)
 
         prompt = template.format(
-            nodes_output=json.dumps(nodes_output, ensure_ascii=False, indent=4),
+            nodes_output=json.dumps(nodes_output, ensure_ascii=False),
             example=example,
             instruction=instruction,
             constraint=FORMAT_CONSTRAINT,
+            prompt_schema="json",
         )
 
         # step2, use `_aask_v1` to get revise structure result
@@ -612,3 +689,32 @@ class ActionNode:
             self.update_instruct_content(revise_contents)
 
         return revise_contents
+
+    @classmethod
+    def from_pydantic(cls, model: Type[BaseModel], key: str = None):
+        """
+        Creates an ActionNode tree from a Pydantic model.
+
+        Args:
+            model (Type[BaseModel]): The Pydantic model to convert.
+
+        Returns:
+            ActionNode: The root node of the created ActionNode tree.
+        """
+        key = key or model.__name__
+        root_node = cls(key=key, expected_type=Type[model], instruction="", example="")
+
+        for field_name, field_info in model.model_fields.items():
+            field_type = field_info.annotation
+            description = field_info.description
+            default = field_info.default
+
+            # Recursively handle nested models if needed
+            if not isinstance(field_type, typing._GenericAlias) and issubclass(field_type, BaseModel):
+                child_node = cls.from_pydantic(field_type, key=field_name)
+            else:
+                child_node = cls(key=field_name, expected_type=field_type, instruction=description, example=default)
+
+            root_node.add_child(child_node)
+
+        return root_node

metagpt/actions/ci/ask_review.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from typing import Tuple
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+from metagpt.schema import Message, Plan
+
+
+class ReviewConst:
+    TASK_REVIEW_TRIGGER = "task"
+    CODE_REVIEW_TRIGGER = "code"
+    CONTINUE_WORDS = ["confirm", "continue", "c", "yes", "y"]
+    CHANGE_WORDS = ["change"]
+    EXIT_WORDS = ["exit"]
+    TASK_REVIEW_INSTRUCTION = (
+        f"If you want to change, add, delete a task or merge tasks in the plan, say '{CHANGE_WORDS[0]} task task_id or current task, ... (things to change)' "
+        f"If you confirm the output from the current task and wish to continue, type: {CONTINUE_WORDS[0]}"
+    )
+    CODE_REVIEW_INSTRUCTION = (
+        f"If you want the codes to be rewritten, say '{CHANGE_WORDS[0]} ... (your change advice)' "
+        f"If you want to leave it as is, type: {CONTINUE_WORDS[0]} or {CONTINUE_WORDS[1]}"
+    )
+    EXIT_INSTRUCTION = f"If you want to terminate the process, type: {EXIT_WORDS[0]}"
+
+
+class AskReview(Action):
+    async def run(
+        self, context: list[Message] = [], plan: Plan = None, trigger: str = ReviewConst.TASK_REVIEW_TRIGGER
+    ) -> Tuple[str, bool]:
+        if plan:
+            logger.info("Current overall plan:")
+            logger.info(
+                "\n".join(
+                    [f"{task.task_id}: {task.instruction}, is_finished: {task.is_finished}" for task in plan.tasks]
+                )
+            )
+
+        logger.info("Most recent context:")
+        latest_action = context[-1].cause_by if context and context[-1].cause_by else ""
+        review_instruction = (
+            ReviewConst.TASK_REVIEW_INSTRUCTION
+            if trigger == ReviewConst.TASK_REVIEW_TRIGGER
+            else ReviewConst.CODE_REVIEW_INSTRUCTION
+        )
+        prompt = (
+            f"This is a <{trigger}> review. Please review output from {latest_action}\n"
+            f"{review_instruction}\n"
+            f"{ReviewConst.EXIT_INSTRUCTION}\n"
+            "Please type your review below:\n"
+        )
+
+        rsp = input(prompt)
+
+        if rsp.lower() in ReviewConst.EXIT_WORDS:
+            exit()
+
+        # Confirmation can be one of "confirm", "continue", "c", "yes", "y" exactly, or sentences containing "confirm".
+        # One could say "confirm this task, but change the next task to ..."
+        confirmed = rsp.lower() in ReviewConst.CONTINUE_WORDS or ReviewConst.CONTINUE_WORDS[0] in rsp.lower()
+
+        return rsp, confirmed

metagpt/actions/ci/debug_code.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+from metagpt.actions.ci.write_analysis_code import BaseWriteAnalysisCode
+from metagpt.logs import logger
+from metagpt.schema import Message
+from metagpt.utils.common import create_func_call_config
+
+DEBUG_REFLECTION_EXAMPLE = '''
+Example 1:
+[previous impl]:
+```python
+def add(a: int, b: int) -> int:
+   """
+   Given integers a and b, return the total value of a and b.
+   """
+   return a - b
+```
+
+[runtime Error]:
+Tested passed:
+
+Tests failed:
+assert add(1, 2) == 3 # output: -1
+assert add(1, 2) == 4 # output: -1
+
+[reflection on previous impl]:
+The implementation failed the test cases where the input integers are 1 and 2. The issue arises because the code does not add the two integers together, but instead subtracts the second integer from the first. To fix this issue, we should change the operator from `-` to `+` in the return statement. This will ensure that the function returns the correct output for the given input.
+
+[improved impl]:
+```python
+def add(a: int, b: int) -> int:
+   """
+   Given integers a and b, return the total value of a and b.
+   """
+   return a + b
+```
+'''
+
+REFLECTION_PROMPT = """
+Here is an example for you.
+{debug_example}
+[context]
+{context}
+
+[previous impl]
+{code}
+[runtime Error]
+{runtime_result}
+
+Analysis the error step by step, provide me improve method and code. Remember to follow [context] requirement. Don't forget write code for steps behind the error step.
+[reflection on previous impl]:
+xxx
+"""
+
+CODE_REFLECTION = {
+    "name": "execute_reflection_code",
+    "description": "Execute reflection code.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "reflection": {
+                "type": "string",
+                "description": "Reflection on previous impl.",
+            },
+            "improved_impl": {
+                "type": "string",
+                "description": "Refined code after reflection.",
+            },
+        },
+        "required": ["reflection", "improved_impl"],
+    },
+}
+
+
+class DebugCode(BaseWriteAnalysisCode):
+    async def run(
+        self,
+        context: list[Message] = None,
+        code: str = "",
+        runtime_result: str = "",
+    ) -> str:
+        """
+        Execute the debugging process based on the provided context, code, and runtime_result.
+
+        Args:
+            context (list[Message]): A list of Message objects representing the context.
+            code (str): The code to be debugged.
+            runtime_result (str): The result of the code execution.
+
+        Returns:
+            str: The improved implementation based on the debugging process.
+        """
+
+        info = []
+        reflection_prompt = REFLECTION_PROMPT.format(
+            debug_example=DEBUG_REFLECTION_EXAMPLE,
+            context=context,
+            code=code,
+            runtime_result=runtime_result,
+        )
+        system_prompt = "You are an AI Python assistant. You will be given your previous implementation code of a task, runtime error results, and a hint to change the implementation appropriately. Write your full implementation "
+        info.append(Message(role="system", content=system_prompt))
+        info.append(Message(role="user", content=reflection_prompt))
+
+        tool_config = create_func_call_config(CODE_REFLECTION)
+        reflection = await self.llm.aask_code(messages=info, **tool_config)
+        logger.info(f"reflection is {reflection}")
+
+        return {"code": reflection["improved_impl"]}

metagpt/actions/ci/execute_nb_code.py

@@ -0,0 +1,249 @@
+# -*- encoding: utf-8 -*-
+"""
+@Date    :   2023/11/17 14:22:15
+@Author  :   orange-crow
+@File    :   execute_nb_code.py
+"""
+from __future__ import annotations
+
+import asyncio
+import base64
+import re
+import traceback
+from typing import Literal, Tuple
+
+import nbformat
+from nbclient import NotebookClient
+from nbclient.exceptions import CellTimeoutError, DeadKernelError
+from nbformat import NotebookNode
+from nbformat.v4 import new_code_cell, new_markdown_cell, new_output
+from rich.box import MINIMAL
+from rich.console import Console, Group
+from rich.live import Live
+from rich.markdown import Markdown
+from rich.panel import Panel
+from rich.syntax import Syntax
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+
+
+class ExecuteNbCode(Action):
+    """execute notebook code block, return result to llm, and display it."""
+
+    nb: NotebookNode
+    nb_client: NotebookClient
+    console: Console
+    interaction: str
+    timeout: int = 600
+
+    def __init__(
+        self,
+        nb=nbformat.v4.new_notebook(),
+        timeout=600,
+    ):
+        super().__init__(
+            nb=nb,
+            nb_client=NotebookClient(nb, timeout=timeout),
+            timeout=timeout,
+            console=Console(),
+            interaction=("ipython" if self.is_ipython() else "terminal"),
+        )
+
+    async def build(self):
+        if self.nb_client.kc is None or not await self.nb_client.kc.is_alive():
+            self.nb_client.create_kernel_manager()
+            self.nb_client.start_new_kernel()
+            self.nb_client.start_new_kernel_client()
+
+    async def terminate(self):
+        """kill NotebookClient"""
+        await self.nb_client._async_cleanup_kernel()
+
+    async def reset(self):
+        """reset NotebookClient"""
+        await self.terminate()
+
+        # sleep 1s to wait for the kernel to be cleaned up completely
+        await asyncio.sleep(1)
+        await self.build()
+        self.nb_client = NotebookClient(self.nb, timeout=self.timeout)
+
+    def add_code_cell(self, code: str):
+        self.nb.cells.append(new_code_cell(source=code))
+
+    def add_markdown_cell(self, markdown: str):
+        self.nb.cells.append(new_markdown_cell(source=markdown))
+
+    def _display(self, code: str, language: Literal["python", "markdown"] = "python"):
+        if language == "python":
+            code = Syntax(code, "python", theme="paraiso-dark", line_numbers=True)
+            self.console.print(code)
+        elif language == "markdown":
+            display_markdown(code)
+        else:
+            raise ValueError(f"Only support for python, markdown, but got {language}")
+
+    def add_output_to_cell(self, cell: NotebookNode, output: str):
+        """add outputs of code execution to notebook cell."""
+        if "outputs" not in cell:
+            cell["outputs"] = []
+        else:
+            cell["outputs"].append(new_output(output_type="stream", name="stdout", text=str(output)))
+
+    def parse_outputs(self, outputs: list[str]) -> str:
+        """Parses the outputs received from notebook execution."""
+        assert isinstance(outputs, list)
+        parsed_output = ""
+
+        for i, output in enumerate(outputs):
+            if output["output_type"] == "stream" and not any(
+                tag in output["text"]
+                for tag in ["| INFO     | metagpt", "| ERROR    | metagpt", "| WARNING  | metagpt"]
+            ):
+                parsed_output += output["text"]
+            elif output["output_type"] == "display_data":
+                if "image/png" in output["data"]:
+                    self.show_bytes_figure(output["data"]["image/png"], self.interaction)
+                else:
+                    logger.info(
+                        f"{i}th output['data'] from nbclient outputs dont have image/png, continue next output ..."
+                    )
+            elif output["output_type"] == "execute_result":
+                parsed_output += output["data"]["text/plain"]
+        return parsed_output
+
+    def show_bytes_figure(self, image_base64: str, interaction_type: Literal["ipython", None]):
+        image_bytes = base64.b64decode(image_base64)
+        if interaction_type == "ipython":
+            from IPython.display import Image, display
+
+            display(Image(data=image_bytes))
+        else:
+            import io
+
+            from PIL import Image
+
+            image = Image.open(io.BytesIO(image_bytes))
+            image.show()
+
+    def is_ipython(self) -> bool:
+        try:
+            # 如果在Jupyter Notebook中运行，__file__ 变量不存在
+            from IPython import get_ipython
+
+            if get_ipython() is not None and "IPKernelApp" in get_ipython().config:
+                return True
+            else:
+                return False
+        except NameError:
+            return False
+
+    async def run_cell(self, cell: NotebookNode, cell_index: int) -> Tuple[bool, str]:
+        """set timeout for run code.
+        returns the success or failure of the cell execution, and an optional error message.
+        """
+        try:
+            await self.nb_client.async_execute_cell(cell, cell_index)
+            return True, ""
+        except CellTimeoutError:
+            assert self.nb_client.km is not None
+            await self.nb_client.km.interrupt_kernel()
+            await asyncio.sleep(1)
+            error_msg = "Cell execution timed out: Execution exceeded the time limit and was stopped; consider optimizing your code for better performance."
+            return False, error_msg
+        except DeadKernelError:
+            await self.reset()
+            return False, "DeadKernelError"
+        except Exception:
+            return False, f"{traceback.format_exc()}"
+
+    async def run(self, code: str, language: Literal["python", "markdown"] = "python") -> Tuple[str, bool]:
+        """
+        return the output of code execution, and a success indicator (bool) of code execution.
+        """
+        self._display(code, language)
+
+        if language == "python":
+            # add code to the notebook
+            self.add_code_cell(code=code)
+
+            # build code executor
+            await self.build()
+
+            # run code
+            cell_index = len(self.nb.cells) - 1
+            success, error_message = await self.run_cell(self.nb.cells[-1], cell_index)
+
+            if not success:
+                return truncate(remove_escape_and_color_codes(error_message), is_success=success)
+
+            # code success
+            outputs = self.parse_outputs(self.nb.cells[-1].outputs)
+            outputs, success = truncate(remove_escape_and_color_codes(outputs), is_success=success)
+
+            if "!pip" in outputs:
+                success = False
+
+            return outputs, success
+
+        elif language == "markdown":
+            # add markdown content to markdown cell in a notebook.
+            self.add_markdown_cell(code)
+            # return True, beacuse there is no execution failure for markdown cell.
+            return code, True
+        else:
+            raise ValueError(f"Only support for language: python, markdown, but got {language}, ")
+
+
+def truncate(result: str, keep_len: int = 2000, is_success: bool = True):
+    """对于超出keep_len个字符的result: 执行失败的代码, 展示result后keep_len个字符; 执行成功的代码, 展示result前keep_len个字符。"""
+    if is_success:
+        desc = f"Executed code successfully. Truncated to show only first {keep_len} characters\n"
+    else:
+        desc = f"Executed code failed, please reflect the cause of bug and then debug. Truncated to show only last {keep_len} characters\n"
+
+    if result.strip().startswith("<coroutine object"):
+        result = "Executed code failed, you need use key word 'await' to run a async code."
+        return result, False
+
+    if len(result) > keep_len:
+        result = result[-keep_len:] if not is_success else result[:keep_len]
+        return desc + result, is_success
+
+    return result, is_success
+
+
+def remove_escape_and_color_codes(input_str: str):
+    # 使用正则表达式去除转义字符和颜色代码
+    pattern = re.compile(r"\x1b\[[0-9;]*[mK]")
+    result = pattern.sub("", input_str)
+    return result
+
+
+def display_markdown(content: str):
+    # 使用正则表达式逐个匹配代码块
+    matches = re.finditer(r"```(.+?)```", content, re.DOTALL)
+    start_index = 0
+    content_panels = []
+    # 逐个打印匹配到的文本和代码
+    for match in matches:
+        text_content = content[start_index : match.start()].strip()
+        code_content = match.group(0).strip()[3:-3]  # Remove triple backticks
+
+        if text_content:
+            content_panels.append(Panel(Markdown(text_content), box=MINIMAL))
+
+        if code_content:
+            content_panels.append(Panel(Markdown(f"```{code_content}"), box=MINIMAL))
+        start_index = match.end()
+
+    # 打印剩余文本（如果有）
+    remaining_text = content[start_index:].strip()
+    if remaining_text:
+        content_panels.append(Panel(Markdown(remaining_text), box=MINIMAL))
+
+    # 在Live模式中显示所有Panel
+    with Live(auto_refresh=False, console=Console(), vertical_overflow="visible") as live:
+        live.update(Group(*content_panels))
+        live.refresh()

metagpt/actions/ci/ml_action.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from typing import Tuple
+
+from metagpt.actions import Action
+from metagpt.actions.ci.write_analysis_code import WriteCodeWithTools
+from metagpt.prompts.ci.ml_action import (
+    ML_GENERATE_CODE_PROMPT,
+    ML_TOOL_USAGE_PROMPT,
+    PRINT_DATA_COLUMNS,
+    UPDATE_DATA_COLUMNS,
+)
+from metagpt.prompts.ci.write_analysis_code import CODE_GENERATOR_WITH_TOOLS
+from metagpt.schema import Message, Plan
+from metagpt.utils.common import create_func_call_config, remove_comments
+
+
+class WriteCodeWithToolsML(WriteCodeWithTools):
+    async def run(
+        self,
+        context: list[Message],
+        plan: Plan = None,
+        column_info: str = "",
+        **kwargs,
+    ) -> Tuple[list[Message], str]:
+        # prepare tool schemas and tool-type-specific instruction
+        tool_schemas, tool_type_usage_prompt = await self._prepare_tools(plan=plan)
+
+        # ML-specific variables to be used in prompt
+        finished_tasks = plan.get_finished_tasks()
+        code_context = [remove_comments(task.code) for task in finished_tasks]
+        code_context = "\n\n".join(code_context)
+
+        # prepare prompt depending on tool availability & LLM call
+        if tool_schemas:
+            prompt = ML_TOOL_USAGE_PROMPT.format(
+                user_requirement=plan.goal,
+                history_code=code_context,
+                current_task=plan.current_task.instruction,
+                column_info=column_info,
+                tool_type_usage_prompt=tool_type_usage_prompt,
+                tool_schemas=tool_schemas,
+            )
+
+        else:
+            prompt = ML_GENERATE_CODE_PROMPT.format(
+                user_requirement=plan.goal,
+                history_code=code_context,
+                current_task=plan.current_task.instruction,
+                column_info=column_info,
+                tool_type_usage_prompt=tool_type_usage_prompt,
+            )
+        tool_config = create_func_call_config(CODE_GENERATOR_WITH_TOOLS)
+        rsp = await self.llm.aask_code(prompt, **tool_config)
+
+        # Extra output to be used for potential debugging
+        context = [Message(content=prompt, role="user")]
+
+        return context, rsp
+
+
+class UpdateDataColumns(Action):
+    async def run(self, plan: Plan = None) -> dict:
+        finished_tasks = plan.get_finished_tasks()
+        code_context = [remove_comments(task.code) for task in finished_tasks]
+        code_context = "\n\n".join(code_context)
+        prompt = UPDATE_DATA_COLUMNS.format(history_code=code_context)
+        tool_config = create_func_call_config(PRINT_DATA_COLUMNS)
+        rsp = await self.llm.aask_code(prompt, **tool_config)
+        return rsp

metagpt/actions/ci/write_analysis_code.py

@@ -0,0 +1,155 @@
+# -*- encoding: utf-8 -*-
+"""
+@Date    :   2023/11/20 13:19:39
+@Author  :   orange-crow
+@File    :   write_analysis_code.py
+"""
+from __future__ import annotations
+
+from typing import Tuple
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+from metagpt.prompts.ci.write_analysis_code import (
+    CODE_GENERATOR_WITH_TOOLS,
+    SELECT_FUNCTION_TOOLS,
+    TOOL_RECOMMENDATION_PROMPT,
+    TOOL_USAGE_PROMPT,
+)
+from metagpt.schema import Message, Plan, SystemMessage
+from metagpt.tools import TOOL_REGISTRY
+from metagpt.tools.tool_registry import validate_tool_names
+from metagpt.utils.common import create_func_call_config
+
+
+class BaseWriteAnalysisCode(Action):
+    DEFAULT_SYSTEM_MSG: str = """You are Code Interpreter, a world-class programmer that can complete any goal by executing code. Strictly follow the plan and generate code step by step. Each step of the code will be executed on the user's machine, and the user will provide the code execution results to you.**Notice: The code for the next step depends on the code for the previous step. Must reuse variables in the lastest other code directly, dont creat it again, it is very import for you. Use !pip install in a standalone block to install missing packages.Usually the libraries you need are already installed.Dont check if packages already imported.**"""  # prompt reference: https://github.com/KillianLucas/open-interpreter/blob/v0.1.4/interpreter/system_message.txt
+    # REUSE_CODE_INSTRUCTION = """ATTENTION: DONT include codes from previous tasks in your current code block, include new codes only, DONT repeat codes!"""
+
+    def insert_system_message(self, context: list[Message], system_msg: str = None):
+        system_msg = system_msg or self.DEFAULT_SYSTEM_MSG
+        context.insert(0, SystemMessage(content=system_msg)) if context[0].role != "system" else None
+        return context
+
+    async def run(self, context: list[Message], plan: Plan = None) -> dict:
+        """Run of a code writing action, used in data analysis or modeling
+
+        Args:
+            context (list[Message]): Action output history, source action denoted by Message.cause_by
+            plan (Plan, optional): Overall plan. Defaults to None.
+
+        Returns:
+            dict: code result in the format of {"code": "print('hello world')", "language": "python"}
+        """
+        raise NotImplementedError
+
+
+class WriteCodeWithoutTools(BaseWriteAnalysisCode):
+    """Ask LLM to generate codes purely by itself without local user-defined tools"""
+
+    async def run(self, context: list[Message], plan: Plan = None, system_msg: str = None, **kwargs) -> dict:
+        messages = self.insert_system_message(context, system_msg)
+        rsp = await self.llm.aask_code(messages, **kwargs)
+        return rsp
+
+
+class WriteCodeWithTools(BaseWriteAnalysisCode):
+    """Write code with help of local available tools. Choose tools first, then generate code to use the tools"""
+
+    # selected tools to choose from, listed by their names. An empty list means selection from all tools.
+    selected_tools: list[str] = []
+
+    def _get_tools_by_type(self, tool_type: str) -> dict:
+        """
+        Retreive tools by tool type from registry, but filtered by pre-selected tool list
+
+        Args:
+            tool_type (str): Tool type to retrieve from the registry
+
+        Returns:
+            dict: A dict of tool name to Tool object, representing available tools under the type
+        """
+        candidate_tools = TOOL_REGISTRY.get_tools_by_type(tool_type)
+        if self.selected_tools:
+            candidate_tool_names = set(self.selected_tools) & candidate_tools.keys()
+            candidate_tools = {tool_name: candidate_tools[tool_name] for tool_name in candidate_tool_names}
+        return candidate_tools
+
+    async def _recommend_tool(
+        self,
+        task: str,
+        available_tools: dict,
+    ) -> dict:
+        """
+        Recommend tools for the specified task.
+
+        Args:
+            task (str): the task to recommend tools for
+            available_tools (dict): the available tools description
+
+        Returns:
+            dict: schemas of recommended tools for the specified task
+        """
+        prompt = TOOL_RECOMMENDATION_PROMPT.format(
+            current_task=task,
+            available_tools=available_tools,
+        )
+        tool_config = create_func_call_config(SELECT_FUNCTION_TOOLS)
+        rsp = await self.llm.aask_code(prompt, **tool_config)
+        recommend_tools = rsp["recommend_tools"]
+        logger.info(f"Recommended tools: \n{recommend_tools}")
+
+        # Parses and validates the  recommended tools, for LLM might hallucinate and recommend non-existing tools
+        valid_tools = validate_tool_names(recommend_tools, return_tool_object=True)
+
+        tool_schemas = {tool.name: tool.schemas for tool in valid_tools}
+
+        return tool_schemas
+
+    async def _prepare_tools(self, plan: Plan) -> Tuple[dict, str]:
+        """Prepare tool schemas and usage instructions according to current task
+
+        Args:
+            plan (Plan): The overall plan containing task information.
+
+        Returns:
+            Tuple[dict, str]: A tool schemas ({tool_name: tool_schema_dict}) and a usage prompt for the type of tools selected
+        """
+        # find tool type from task type through exact match, can extend to retrieval in the future
+        tool_type = plan.current_task.task_type
+
+        # prepare tool-type-specific instruction
+        tool_type_usage_prompt = (
+            TOOL_REGISTRY.get_tool_type(tool_type).usage_prompt if TOOL_REGISTRY.has_tool_type(tool_type) else ""
+        )
+
+        # prepare schemas of available tools
+        tool_schemas = {}
+        available_tools = self._get_tools_by_type(tool_type)
+        if available_tools:
+            available_tools = {tool_name: tool.schemas["description"] for tool_name, tool in available_tools.items()}
+            tool_schemas = await self._recommend_tool(plan.current_task.instruction, available_tools)
+
+        return tool_schemas, tool_type_usage_prompt
+
+    async def run(
+        self,
+        context: list[Message],
+        plan: Plan,
+        **kwargs,
+    ) -> str:
+        # prepare tool schemas and tool-type-specific instruction
+        tool_schemas, tool_type_usage_prompt = await self._prepare_tools(plan=plan)
+
+        # form a complete tool usage instruction and include it as a message in context
+        tools_instruction = TOOL_USAGE_PROMPT.format(
+            tool_schemas=tool_schemas, tool_type_usage_prompt=tool_type_usage_prompt
+        )
+        context.append(Message(content=tools_instruction, role="user"))
+
+        # prepare prompt & LLM call
+        prompt = self.insert_system_message(context)
+        tool_config = create_func_call_config(CODE_GENERATOR_WITH_TOOLS)
+        rsp = await self.llm.aask_code(prompt, **tool_config)
+
+        return rsp

metagpt/actions/ci/write_plan.py

@@ -0,0 +1,116 @@
+# -*- encoding: utf-8 -*-
+"""
+@Date    :   2023/11/20 11:24:03
+@Author  :   orange-crow
+@File    :   plan.py
+"""
+from __future__ import annotations
+
+import json
+from copy import deepcopy
+from typing import Tuple
+
+from metagpt.actions import Action
+from metagpt.logs import logger
+from metagpt.prompts.ci.write_analysis_code import (
+    ASSIGN_TASK_TYPE_CONFIG,
+    ASSIGN_TASK_TYPE_PROMPT,
+)
+from metagpt.schema import Message, Plan, Task
+from metagpt.tools import TOOL_REGISTRY
+from metagpt.utils.common import CodeParser, create_func_call_config
+
+
+class WritePlan(Action):
+    PROMPT_TEMPLATE: str = """
+    # Context:
+    __context__
+    # Task:
+    Based on the context, write a plan or modify an existing plan of what you should do to achieve the goal. A plan consists of one to __max_tasks__ tasks.
+    If you are modifying an existing plan, carefully follow the instruction, don't make unnecessary changes. Give the whole plan unless instructed to modify only one task of the plan.
+    If you encounter errors on the current task, revise and output the current single task only.
+    Output a list of jsons following the format:
+    ```json
+    [
+        {
+            "task_id": str = "unique identifier for a task in plan, can be an ordinal",
+            "dependent_task_ids": list[str] = "ids of tasks prerequisite to this task",
+            "instruction": "what you should do in this task, one short phrase or sentence",
+        },
+        ...
+    ]
+    ```
+    """
+
+    async def assign_task_type(self, tasks: list[dict]) -> str:
+        """Assign task type to each task in tasks
+
+        Args:
+            tasks (list[dict]): tasks to be assigned task type
+
+        Returns:
+            str: tasks with task type assigned in a json string
+        """
+        task_info = "\n".join([f"Task {task['task_id']}: {task['instruction']}" for task in tasks])
+        task_type_desc = "\n".join(
+            [f"- **{tool_type.name}**: {tool_type.desc}" for tool_type in TOOL_REGISTRY.get_tool_types().values()]
+        )  # task type are binded with tool type now, should be improved in the future
+        prompt = ASSIGN_TASK_TYPE_PROMPT.format(
+            task_info=task_info, task_type_desc=task_type_desc
+        )  # task types are set to be the same as tool types, for now
+        tool_config = create_func_call_config(ASSIGN_TASK_TYPE_CONFIG)
+        rsp = await self.llm.aask_code(prompt, **tool_config)
+        task_type_list = rsp["task_type"]
+        logger.info(f"assigned task types: {task_type_list}")
+        for task, task_type in zip(tasks, task_type_list):
+            task["task_type"] = task_type
+        return json.dumps(tasks)
+
+    async def run(self, context: list[Message], max_tasks: int = 5, use_tools: bool = False) -> str:
+        prompt = (
+            self.PROMPT_TEMPLATE.replace("__context__", "\n".join([str(ct) for ct in context]))
+            # .replace("__current_plan__", current_plan)
+            .replace("__max_tasks__", str(max_tasks))
+        )
+        rsp = await self._aask(prompt)
+        rsp = CodeParser.parse_code(block=None, text=rsp)
+        if use_tools:
+            rsp = await self.assign_task_type(json.loads(rsp))
+        return rsp
+
+
+def rsp_to_tasks(rsp: str) -> list[Task]:
+    rsp = json.loads(rsp)
+    tasks = [Task(**task_config) for task_config in rsp]
+    return tasks
+
+
+def update_plan_from_rsp(rsp: str, current_plan: Plan):
+    tasks = rsp_to_tasks(rsp)
+    if len(tasks) == 1 or tasks[0].dependent_task_ids:
+        if tasks[0].dependent_task_ids and len(tasks) > 1:
+            # tasks[0].dependent_task_ids means the generated tasks are not a complete plan
+            # for they depend on tasks in the current plan, in this case, we only support updating one task each time
+            logger.warning(
+                "Current plan will take only the first generated task if the generated tasks are not a complete plan"
+            )
+        # handle a single task
+        if current_plan.has_task_id(tasks[0].task_id):
+            # replace an existing task
+            current_plan.replace_task(tasks[0])
+        else:
+            # append one task
+            current_plan.append_task(tasks[0])
+
+    else:
+        # add tasks in general
+        current_plan.add_tasks(tasks)
+
+
+def precheck_update_plan_from_rsp(rsp: str, current_plan: Plan) -> Tuple[bool, str]:
+    temp_plan = deepcopy(current_plan)
+    try:
+        update_plan_from_rsp(rsp, temp_plan)
+        return True, ""
+    except Exception as e:
+        return False, e

metagpt/actions/debug_error.py

@@ -49,7 +49,7 @@ class DebugError(Action):
     i_context: RunCodeContext = Field(default_factory=RunCodeContext)
 
     async def run(self, *args, **kwargs) -> str:
-        output_doc = await self.project_repo.test_outputs.get(filename=self.i_context.output_filename)
+        output_doc = await self.repo.test_outputs.get(filename=self.i_context.output_filename)
         if not output_doc:
             return ""
         output_detail = RunCodeResult.loads(output_doc.content)
@@ -59,12 +59,12 @@ class DebugError(Action):
             return ""
 
         logger.info(f"Debug and rewrite {self.i_context.test_filename}")
-        code_doc = await self.project_repo.with_src_path(self.context.src_workspace).srcs.get(
+        code_doc = await self.repo.with_src_path(self.context.src_workspace).srcs.get(
             filename=self.i_context.code_filename
         )
         if not code_doc:
             return ""
-        test_doc = await self.project_repo.tests.get(filename=self.i_context.test_filename)
+        test_doc = await self.repo.tests.get(filename=self.i_context.test_filename)
         if not test_doc:
             return ""
         prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr)

metagpt/actions/design_api.py

@@ -14,7 +14,14 @@ from pathlib import Path
 from typing import Optional
 
 from metagpt.actions import Action, ActionOutput
-from metagpt.actions.design_api_an import DESIGN_API_NODE
+from metagpt.actions.design_api_an import (
+    DATA_STRUCTURES_AND_INTERFACES,
+    DESIGN_API_NODE,
+    PROGRAM_CALL_FLOW,
+    REFINED_DATA_STRUCTURES_AND_INTERFACES,
+    REFINED_DESIGN_NODE,
+    REFINED_PROGRAM_CALL_FLOW,
+)
 from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO
 from metagpt.logs import logger
 from metagpt.schema import Document, Documents, Message
@@ -39,11 +46,11 @@ class WriteDesign(Action):
     )
 
     async def run(self, with_messages: Message, schema: str = None):
-        # Use `git status` to identify which PRD documents have been modified in the `docs/prds` directory.
-        changed_prds = self.project_repo.docs.prd.changed_files
+        # Use `git status` to identify which PRD documents have been modified in the `docs/prd` directory.
+        changed_prds = self.repo.docs.prd.changed_files
         # Use `git status` to identify which design documents in the `docs/system_designs` directory have undergone
         # changes.
-        changed_system_designs = self.project_repo.docs.system_design.changed_files
+        changed_system_designs = self.repo.docs.system_design.changed_files
 
         # For those PRDs and design documents that have undergone changes, regenerate the design content.
         changed_files = Documents()
@@ -68,46 +75,46 @@ class WriteDesign(Action):
 
     async def _merge(self, prd_doc, system_design_doc):
         context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content)
-        node = await DESIGN_API_NODE.fill(context=context, llm=self.llm)
+        node = await REFINED_DESIGN_NODE.fill(context=context, llm=self.llm)
         system_design_doc.content = node.instruct_content.model_dump_json()
         return system_design_doc
 
     async def _update_system_design(self, filename) -> Document:
-        prd = await self.project_repo.docs.prd.get(filename)
-        old_system_design_doc = await self.project_repo.docs.system_design.get(filename)
+        prd = await self.repo.docs.prd.get(filename)
+        old_system_design_doc = await self.repo.docs.system_design.get(filename)
         if not old_system_design_doc:
             system_design = await self._new_system_design(context=prd.content)
-            doc = await self.project_repo.docs.system_design.save(
+            doc = await self.repo.docs.system_design.save(
                 filename=filename,
                 content=system_design.instruct_content.model_dump_json(),
                 dependencies={prd.root_relative_path},
             )
         else:
             doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc)
-            await self.project_repo.docs.system_design.save_doc(doc=doc, dependencies={prd.root_relative_path})
+            await self.repo.docs.system_design.save_doc(doc=doc, dependencies={prd.root_relative_path})
         await self._save_data_api_design(doc)
         await self._save_seq_flow(doc)
-        await self.project_repo.resources.system_design.save_pdf(doc=doc)
+        await self.repo.resources.system_design.save_pdf(doc=doc)
         return doc
 
     async def _save_data_api_design(self, design_doc):
         m = json.loads(design_doc.content)
-        data_api_design = m.get("Data structures and interfaces")
+        data_api_design = m.get(DATA_STRUCTURES_AND_INTERFACES.key) or m.get(REFINED_DATA_STRUCTURES_AND_INTERFACES.key)
         if not data_api_design:
             return
-        pathname = self.project_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("")
+        pathname = self.repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("")
         await self._save_mermaid_file(data_api_design, pathname)
         logger.info(f"Save class view to {str(pathname)}")
 
     async def _save_seq_flow(self, design_doc):
         m = json.loads(design_doc.content)
-        seq_flow = m.get("Program call flow")
+        seq_flow = m.get(PROGRAM_CALL_FLOW.key) or m.get(REFINED_PROGRAM_CALL_FLOW.key)
         if not seq_flow:
             return
-        pathname = self.project_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("")
+        pathname = self.repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("")
         await self._save_mermaid_file(seq_flow, pathname)
         logger.info(f"Saving sequence flow to {str(pathname)}")
 
     async def _save_mermaid_file(self, data: str, pathname: Path):
         pathname.parent.mkdir(parents=True, exist_ok=True)
-        await mermaid_to_file(self.config.mermaid_engine, data, pathname)
+        await mermaid_to_file(self.config.mermaid.engine, data, pathname)

metagpt/actions/design_api_an.py

@@ -8,6 +8,7 @@
 from typing import List
 
 from metagpt.actions.action_node import ActionNode
+from metagpt.logs import logger
 from metagpt.utils.mermaid import MMC1, MMC2
 
 IMPLEMENTATION_APPROACH = ActionNode(
@@ -17,6 +18,15 @@ IMPLEMENTATION_APPROACH = ActionNode(
     example="We will ...",
 )
 
+REFINED_IMPLEMENTATION_APPROACH = ActionNode(
+    key="Refined Implementation Approach",
+    expected_type=str,
+    instruction="Update and extend the original implementation approach to reflect the evolving challenges and "
+    "requirements due to incremental development. Outline the steps involved in the implementation process with the "
+    "detailed strategies.",
+    example="We will refine ...",
+)
+
 PROJECT_NAME = ActionNode(
     key="Project name", expected_type=str, instruction="The project name with underline", example="game_2048"
 )
@@ -28,6 +38,14 @@ FILE_LIST = ActionNode(
     example=["main.py", "game.py"],
 )
 
+REFINED_FILE_LIST = ActionNode(
+    key="Refined File list",
+    expected_type=List[str],
+    instruction="Update and expand the original file list including only relative paths. Up to 2 files can be added."
+    "Ensure that the refined file list reflects the evolving structure of the project.",
+    example=["main.py", "game.py", "new_feature.py"],
+)
+
 DATA_STRUCTURES_AND_INTERFACES = ActionNode(
     key="Data structures and interfaces",
     expected_type=str,
@@ -37,6 +55,16 @@ DATA_STRUCTURES_AND_INTERFACES = ActionNode(
     example=MMC1,
 )
 
+REFINED_DATA_STRUCTURES_AND_INTERFACES = ActionNode(
+    key="Refined Data structures and interfaces",
+    expected_type=str,
+    instruction="Update and extend the existing mermaid classDiagram code syntax to incorporate new classes, "
+    "methods (including __init__), and functions with precise type annotations. Delineate additional "
+    "relationships between classes, ensuring clarity and adherence to PEP8 standards."
+    "Retain content that is not related to incremental development but important for consistency and clarity.",
+    example=MMC1,
+)
+
 PROGRAM_CALL_FLOW = ActionNode(
     key="Program call flow",
     expected_type=str,
@@ -45,6 +73,16 @@ PROGRAM_CALL_FLOW = ActionNode(
     example=MMC2,
 )
 
+REFINED_PROGRAM_CALL_FLOW = ActionNode(
+    key="Refined Program call flow",
+    expected_type=str,
+    instruction="Extend the existing sequenceDiagram code syntax with detailed information, accurately covering the"
+    "CRUD and initialization of each object. Ensure correct syntax usage and reflect the incremental changes introduced"
+    "in the classes and API defined above. "
+    "Retain content that is not related to incremental development but important for consistency and clarity.",
+    example=MMC2,
+)
+
 ANYTHING_UNCLEAR = ActionNode(
     key="Anything UNCLEAR",
     expected_type=str,
@@ -61,4 +99,24 @@ NODES = [
     ANYTHING_UNCLEAR,
 ]
 
+REFINED_NODES = [
+    REFINED_IMPLEMENTATION_APPROACH,
+    REFINED_FILE_LIST,
+    REFINED_DATA_STRUCTURES_AND_INTERFACES,
+    REFINED_PROGRAM_CALL_FLOW,
+    ANYTHING_UNCLEAR,
+]
+
 DESIGN_API_NODE = ActionNode.from_children("DesignAPI", NODES)
+REFINED_DESIGN_NODE = ActionNode.from_children("RefinedDesignAPI", REFINED_NODES)
+
+
+def main():
+    prompt = DESIGN_API_NODE.compile(context="")
+    logger.info(prompt)
+    prompt = REFINED_DESIGN_NODE.compile(context="")
+    logger.info(prompt)
+
+
+if __name__ == "__main__":
+    main()

metagpt/actions/prepare_documents.py

@@ -15,6 +15,7 @@ from metagpt.actions import Action, ActionOutput
 from metagpt.const import REQUIREMENT_FILENAME
 from metagpt.utils.file_repository import FileRepository
 from metagpt.utils.git_repository import GitRepository
+from metagpt.utils.project_repo import ProjectRepo
 
 
 class PrepareDocuments(Action):
@@ -38,13 +39,14 @@ class PrepareDocuments(Action):
             shutil.rmtree(path)
         self.config.project_path = path
         self.context.git_repo = GitRepository(local_path=path, auto_init=True)
+        self.context.repo = ProjectRepo(self.context.git_repo)
 
     async def run(self, with_messages, **kwargs):
         """Create and initialize the workspace folder, initialize the Git environment."""
         self._init_repo()
 
         # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`.
-        doc = await self.project_repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
+        doc = await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
         # Send a Message notification to the WritePRD action, instructing it to process requirements using
-        # `docs/requirement.txt` and `docs/prds/`.
+        # `docs/requirement.txt` and `docs/prd/`.
         return ActionOutput(content=doc.content, instruct_content=doc)

metagpt/actions/project_management.py

@@ -13,16 +13,16 @@
 import json
 from typing import Optional
 
-from metagpt.actions import ActionOutput
 from metagpt.actions.action import Action
-from metagpt.actions.project_management_an import PM_NODE
+from metagpt.actions.action_output import ActionOutput
+from metagpt.actions.project_management_an import PM_NODE, REFINED_PM_NODE
 from metagpt.const import PACKAGE_REQUIREMENTS_FILENAME
 from metagpt.logs import logger
 from metagpt.schema import Document, Documents
 
 NEW_REQ_TEMPLATE = """
 ### Legacy Content
-{old_tasks}
+{old_task}
 
 ### New Requirements
 {context}
@@ -34,8 +34,8 @@ class WriteTasks(Action):
     i_context: Optional[str] = None
 
     async def run(self, with_messages):
-        changed_system_designs = self.project_repo.docs.system_design.changed_files
-        changed_tasks = self.project_repo.docs.task.changed_files
+        changed_system_designs = self.repo.docs.system_design.changed_files
+        changed_tasks = self.repo.docs.task.changed_files
         change_files = Documents()
         # Rewrite the system designs that have undergone changes based on the git head diff under
         # `docs/system_designs/`.
@@ -57,16 +57,14 @@ class WriteTasks(Action):
         return ActionOutput(content=change_files.model_dump_json(), instruct_content=change_files)
 
     async def _update_tasks(self, filename):
-        system_design_doc = await self.project_repo.docs.system_design.get(filename)
-        task_doc = await self.project_repo.docs.task.get(filename)
+        system_design_doc = await self.repo.docs.system_design.get(filename)
+        task_doc = await self.repo.docs.task.get(filename)
         if task_doc:
             task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc)
-            await self.project_repo.docs.task.save_doc(
-                doc=task_doc, dependencies={system_design_doc.root_relative_path}
-            )
+            await self.repo.docs.task.save_doc(doc=task_doc, dependencies={system_design_doc.root_relative_path})
         else:
             rsp = await self._run_new_tasks(context=system_design_doc.content)
-            task_doc = await self.project_repo.docs.task.save(
+            task_doc = await self.repo.docs.task.save(
                 filename=filename,
                 content=rsp.instruct_content.model_dump_json(),
                 dependencies={system_design_doc.root_relative_path},
@@ -79,15 +77,15 @@ class WriteTasks(Action):
         return node
 
     async def _merge(self, system_design_doc, task_doc) -> Document:
-        context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_tasks=task_doc.content)
-        node = await PM_NODE.fill(context, self.llm, schema=self.prompt_schema)
+        context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_task=task_doc.content)
+        node = await REFINED_PM_NODE.fill(context, self.llm, schema=self.prompt_schema)
         task_doc.content = node.instruct_content.model_dump_json()
         return task_doc
 
     async def _update_requirements(self, doc):
         m = json.loads(doc.content)
-        packages = set(m.get("Required Python third-party packages", set()))
-        requirement_doc = await self.project_repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME)
+        packages = set(m.get("Required Python packages", set()))
+        requirement_doc = await self.repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME)
         if not requirement_doc:
             requirement_doc = Document(filename=PACKAGE_REQUIREMENTS_FILENAME, root_path=".", content="")
         lines = requirement_doc.content.splitlines()
@@ -95,4 +93,4 @@ class WriteTasks(Action):
             if pkg == "":
                 continue
             packages.add(pkg)
-        await self.project_repo.save(filename=PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages))
+        await self.repo.save(filename=PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages))

metagpt/actions/project_management_an.py

@@ -35,6 +35,20 @@ LOGIC_ANALYSIS = ActionNode(
     ],
 )
 
+REFINED_LOGIC_ANALYSIS = ActionNode(
+    key="Refined Logic Analysis",
+    expected_type=List[List[str]],
+    instruction="Review and refine the logic analysis by merging the Legacy Content and Incremental Content. "
+    "Provide a comprehensive list of files with classes/methods/functions to be implemented or modified incrementally. "
+    "Include dependency analysis, consider potential impacts on existing code, and document necessary imports.",
+    example=[
+        ["game.py", "Contains Game class and ... functions"],
+        ["main.py", "Contains main function, from game import Game"],
+        ["new_feature.py", "Introduces NewFeature class and related functions"],
+        ["utils.py", "Modifies existing utility functions to support incremental changes"],
+    ],
+)
+
 TASK_LIST = ActionNode(
     key="Task list",
     expected_type=List[str],
@@ -42,6 +56,15 @@ TASK_LIST = ActionNode(
     example=["game.py", "main.py"],
 )
 
+REFINED_TASK_LIST = ActionNode(
+    key="Refined Task list",
+    expected_type=List[str],
+    instruction="Review and refine the combined task list after the merger of Legacy Content and Incremental Content, "
+    "and consistent with Refined File List. Ensure that tasks are organized in a logical and prioritized order, "
+    "considering dependencies for a streamlined and efficient development process. ",
+    example=["new_feature.py", "utils", "game.py", "main.py"],
+)
+
 FULL_API_SPEC = ActionNode(
     key="Full API spec",
     expected_type=str,
@@ -54,9 +77,19 @@ SHARED_KNOWLEDGE = ActionNode(
     key="Shared Knowledge",
     expected_type=str,
     instruction="Detail any shared knowledge, like common utility functions or configuration variables.",
-    example="'game.py' contains functions shared across the project.",
+    example="`game.py` contains functions shared across the project.",
 )
 
+REFINED_SHARED_KNOWLEDGE = ActionNode(
+    key="Refined Shared Knowledge",
+    expected_type=str,
+    instruction="Update and expand shared knowledge to reflect any new elements introduced. This includes common "
+    "utility functions, configuration variables for team collaboration. Retain content that is not related to "
+    "incremental development but important for consistency and clarity.",
+    example="`new_module.py` enhances shared utility functions for improved code reusability and collaboration.",
+)
+
+
 ANYTHING_UNCLEAR_PM = ActionNode(
     key="Anything UNCLEAR",
     expected_type=str,
@@ -74,13 +107,25 @@ NODES = [
     ANYTHING_UNCLEAR_PM,
 ]
 
+REFINED_NODES = [
+    REQUIRED_PYTHON_PACKAGES,
+    REQUIRED_OTHER_LANGUAGE_PACKAGES,
+    REFINED_LOGIC_ANALYSIS,
+    REFINED_TASK_LIST,
+    FULL_API_SPEC,
+    REFINED_SHARED_KNOWLEDGE,
+    ANYTHING_UNCLEAR_PM,
+]
 
 PM_NODE = ActionNode.from_children("PM_NODE", NODES)
+REFINED_PM_NODE = ActionNode.from_children("REFINED_PM_NODE", REFINED_NODES)
 
 
 def main():
     prompt = PM_NODE.compile(context="")
     logger.info(prompt)
+    prompt = REFINED_PM_NODE.compile(context="")
+    logger.info(prompt)
 
 
 if __name__ == "__main__":

metagpt/actions/research.py

@@ -3,15 +3,15 @@
 from __future__ import annotations
 
 import asyncio
-from typing import Callable, Optional, Union
+from typing import Any, Callable, Optional, Union
 
-from pydantic import Field, parse_obj_as
+from pydantic import TypeAdapter, model_validator
 
 from metagpt.actions import Action
 from metagpt.config2 import config
 from metagpt.logs import logger
 from metagpt.tools.search_engine import SearchEngine
-from metagpt.tools.web_browser_engine import WebBrowserEngine, WebBrowserEngineType
+from metagpt.tools.web_browser_engine import WebBrowserEngine
 from metagpt.utils.common import OutputParser
 from metagpt.utils.text import generate_prompt_chunk, reduce_message_length
 
@@ -81,10 +81,16 @@ class CollectLinks(Action):
     name: str = "CollectLinks"
     i_context: Optional[str] = None
     desc: str = "Collect links from a search engine."
-
-    search_engine: SearchEngine = Field(default_factory=SearchEngine)
+    search_func: Optional[Any] = None
+    search_engine: Optional[SearchEngine] = None
     rank_func: Optional[Callable[[list[str]], None]] = None
 
+    @model_validator(mode="after")
+    def validate_engine_and_run_func(self):
+        if self.search_engine is None:
+            self.search_engine = SearchEngine.from_search_config(self.config.search, proxy=self.config.proxy)
+        return self
+
     async def run(
         self,
         topic: str,
@@ -107,7 +113,7 @@ class CollectLinks(Action):
         keywords = await self._aask(SEARCH_TOPIC_PROMPT, [system_text])
         try:
             keywords = OutputParser.extract_struct(keywords, list)
-            keywords = parse_obj_as(list[str], keywords)
+            keywords = TypeAdapter(list[str]).validate_python(keywords)
         except Exception as e:
             logger.exception(f"fail to get keywords related to the research topic '{topic}' for {e}")
             keywords = [topic]
@@ -127,13 +133,13 @@ class CollectLinks(Action):
                 if len(remove) == 0:
                     break
 
-        model_name = config.get_openai_llm().model
+        model_name = config.llm.model
         prompt = reduce_message_length(gen_msg(), model_name, system_text, 4096)
         logger.debug(prompt)
         queries = await self._aask(prompt, [system_text])
         try:
             queries = OutputParser.extract_struct(queries, list)
-            queries = parse_obj_as(list[str], queries)
+            queries = TypeAdapter(list[str]).validate_python(queries)
         except Exception as e:
             logger.exception(f"fail to break down the research question due to {e}")
             queries = keywords
@@ -178,15 +184,17 @@ class WebBrowseAndSummarize(Action):
     i_context: Optional[str] = None
     desc: str = "Explore the web and provide summaries of articles and webpages."
     browse_func: Union[Callable[[list[str]], None], None] = None
-    web_browser_engine: Optional[WebBrowserEngine] = WebBrowserEngineType.PLAYWRIGHT
+    web_browser_engine: Optional[WebBrowserEngine] = None
 
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-
-        self.web_browser_engine = WebBrowserEngine(
-            engine=WebBrowserEngineType.CUSTOM if self.browse_func else WebBrowserEngineType.PLAYWRIGHT,
-            run_func=self.browse_func,
-        )
+    @model_validator(mode="after")
+    def validate_engine_and_run_func(self):
+        if self.web_browser_engine is None:
+            self.web_browser_engine = WebBrowserEngine.from_browser_config(
+                self.config.browser,
+                browse_func=self.browse_func,
+                proxy=self.config.proxy,
+            )
+        return self
 
     async def run(
         self,

metagpt/actions/run_code.py

@@ -16,6 +16,7 @@
             class.
 """
 import subprocess
+from pathlib import Path
 from typing import Tuple
 
 from pydantic import Field
@@ -41,13 +42,13 @@ Determine the ONE file to rewrite in order to fix the error, for example, xyz.py
 Determine if all of the code works fine, if so write PASS, else FAIL,
 WRITE ONLY ONE WORD, PASS OR FAIL, IN THIS SECTION
 ## Send To:
-Please write Engineer if the errors are due to problematic development codes, and QaEngineer to problematic test codes, and NoOne if there are no errors,
-WRITE ONLY ONE WORD, Engineer OR QaEngineer OR NoOne, IN THIS SECTION.
+Please write NoOne if there are no errors, Engineer if the errors are due to problematic development codes, else QaEngineer,
+WRITE ONLY ONE WORD, NoOne OR Engineer OR QaEngineer, IN THIS SECTION.
 ---
 You should fill in necessary instruction, status, send to, and finally return all content between the --- segment line.
 """
 
-CONTEXT = """
+TEMPLATE_CONTEXT = """
 ## Development Code File Name
 {code_file_name}
 ## Development Code
@@ -130,7 +131,7 @@ class RunCode(Action):
         logger.info(f"{outs=}")
         logger.info(f"{errs=}")
 
-        context = CONTEXT.format(
+        context = TEMPLATE_CONTEXT.format(
             code=self.i_context.code,
             code_file_name=self.i_context.code_filename,
             test_code=self.i_context.test_code,
@@ -150,11 +151,23 @@ class RunCode(Action):
         return subprocess.run(cmd, check=check, cwd=cwd, env=env)
 
     @staticmethod
-    def _install_dependencies(working_directory, env):
+    def _install_requirements(working_directory, env):
+        file_path = Path(working_directory) / "requirements.txt"
+        if not file_path.exists():
+            return
+        if file_path.stat().st_size == 0:
+            return
         install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"]
         logger.info(" ".join(install_command))
         RunCode._install_via_subprocess(install_command, check=True, cwd=working_directory, env=env)
 
+    @staticmethod
+    def _install_pytest(working_directory, env):
         install_pytest_command = ["python", "-m", "pip", "install", "pytest"]
         logger.info(" ".join(install_pytest_command))
         RunCode._install_via_subprocess(install_pytest_command, check=True, cwd=working_directory, env=env)
+
+    @staticmethod
+    def _install_dependencies(working_directory, env):
+        RunCode._install_requirements(working_directory, env)
+        RunCode._install_pytest(working_directory, env)

metagpt/actions/search_and_summarize.py

@@ -5,7 +5,7 @@
 @Author  : alexanderwu
 @File    : search_google.py
 """
-from typing import Any, Optional
+from typing import Optional
 
 import pydantic
 from pydantic import model_validator
@@ -13,7 +13,6 @@ from pydantic import model_validator
 from metagpt.actions import Action
 from metagpt.logs import logger
 from metagpt.schema import Message
-from metagpt.tools import SearchEngineType
 from metagpt.tools.search_engine import SearchEngine
 
 SEARCH_AND_SUMMARIZE_SYSTEM = """### Requirements
@@ -105,21 +104,19 @@ You are a member of a professional butler team and will provide helpful suggesti
 class SearchAndSummarize(Action):
     name: str = ""
     content: Optional[str] = None
-    engine: Optional[SearchEngineType] = None
-    search_func: Optional[Any] = None
     search_engine: SearchEngine = None
     result: str = ""
 
     @model_validator(mode="after")
-    def validate_engine_and_run_func(self):
-        if self.engine is None:
-            self.engine = self.config.search_engine
-        try:
-            search_engine = SearchEngine(engine=self.engine, run_func=self.search_func)
-        except pydantic.ValidationError:
-            search_engine = None
+    def validate_search_engine(self):
+        if self.search_engine is None:
+            try:
+                config = self.config
+                search_engine = SearchEngine.from_search_config(config.search, proxy=config.proxy)
+            except pydantic.ValidationError:
+                search_engine = None
 
-        self.search_engine = search_engine
+            self.search_engine = search_engine
         return self
 
     async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str:

metagpt/actions/summarize_code.py

@@ -26,9 +26,9 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
 {system_design}
 ```
 -----
-# Tasks
+# Task
 ```text
-{tasks}
+{task}
 ```
 -----
 {code_blocks}
@@ -98,10 +98,10 @@ class SummarizeCode(Action):
 
     async def run(self):
         design_pathname = Path(self.i_context.design_filename)
-        design_doc = await self.project_repo.docs.system_design.get(filename=design_pathname.name)
+        design_doc = await self.repo.docs.system_design.get(filename=design_pathname.name)
         task_pathname = Path(self.i_context.task_filename)
-        task_doc = await self.project_repo.docs.task.get(filename=task_pathname.name)
-        src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs
+        task_doc = await self.repo.docs.task.get(filename=task_pathname.name)
+        src_file_repo = self.repo.with_src_path(self.context.src_workspace).srcs
         code_blocks = []
         for filename in self.i_context.codes_filenames:
             code_doc = await src_file_repo.get(filename)
@@ -110,7 +110,7 @@ class SummarizeCode(Action):
         format_example = FORMAT_EXAMPLE
         prompt = PROMPT_TEMPLATE.format(
             system_design=design_doc.content,
-            tasks=task_doc.content,
+            task=task_doc.content,
             code_blocks="\n".join(code_blocks),
             format_example=format_example,
         )

metagpt/actions/write_code.py

@@ -21,10 +21,17 @@ from pydantic import Field
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions.action import Action
-from metagpt.const import BUGFIX_FILENAME
+from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST
+from metagpt.actions.write_code_plan_and_change_an import REFINED_TEMPLATE
+from metagpt.const import (
+    BUGFIX_FILENAME,
+    CODE_PLAN_AND_CHANGE_FILENAME,
+    REQUIREMENT_FILENAME,
+)
 from metagpt.logs import logger
 from metagpt.schema import CodingContext, Document, RunCodeResult
 from metagpt.utils.common import CodeParser
+from metagpt.utils.project_repo import ProjectRepo
 
 PROMPT_TEMPLATE = """
 NOTICE
@@ -36,8 +43,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
 ## Design
 {design}
 
-## Tasks
-{tasks}
+## Task
+{task}
 
 ## Legacy Code
 ```Code
@@ -88,12 +95,15 @@ class WriteCode(Action):
         return code
 
     async def run(self, *args, **kwargs) -> CodingContext:
-        bug_feedback = await self.project_repo.docs.get(filename=BUGFIX_FILENAME)
+        bug_feedback = await self.repo.docs.get(filename=BUGFIX_FILENAME)
         coding_context = CodingContext.loads(self.i_context.content)
-        test_doc = await self.project_repo.test_outputs.get(filename="test_" + coding_context.filename + ".json")
+        test_doc = await self.repo.test_outputs.get(filename="test_" + coding_context.filename + ".json")
+        code_plan_and_change_doc = await self.repo.docs.code_plan_and_change.get(filename=CODE_PLAN_AND_CHANGE_FILENAME)
+        code_plan_and_change = code_plan_and_change_doc.content if code_plan_and_change_doc else ""
+        requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME)
         summary_doc = None
         if coding_context.design_doc and coding_context.design_doc.filename:
-            summary_doc = await self.project_repo.docs.code_summary.get(filename=coding_context.design_doc.filename)
+            summary_doc = await self.repo.docs.code_summary.get(filename=coding_context.design_doc.filename)
         logs = ""
         if test_doc:
             test_detail = RunCodeResult.loads(test_doc.content)
@@ -101,22 +111,39 @@ class WriteCode(Action):
 
         if bug_feedback:
             code_context = coding_context.code_doc.content
+        elif code_plan_and_change:
+            code_context = await self.get_codes(
+                coding_context.task_doc, exclude=self.i_context.filename, project_repo=self.repo, use_inc=True
+            )
         else:
             code_context = await self.get_codes(
                 coding_context.task_doc,
                 exclude=self.i_context.filename,
-                project_repo=self.project_repo.with_src_path(self.context.src_workspace),
+                project_repo=self.repo.with_src_path(self.context.src_workspace),
             )
 
-        prompt = PROMPT_TEMPLATE.format(
-            design=coding_context.design_doc.content if coding_context.design_doc else "",
-            tasks=coding_context.task_doc.content if coding_context.task_doc else "",
-            code=code_context,
-            logs=logs,
-            feedback=bug_feedback.content if bug_feedback else "",
-            filename=self.i_context.filename,
-            summary_log=summary_doc.content if summary_doc else "",
-        )
+        if code_plan_and_change:
+            prompt = REFINED_TEMPLATE.format(
+                user_requirement=requirement_doc.content if requirement_doc else "",
+                code_plan_and_change=code_plan_and_change,
+                design=coding_context.design_doc.content if coding_context.design_doc else "",
+                task=coding_context.task_doc.content if coding_context.task_doc else "",
+                code=code_context,
+                logs=logs,
+                feedback=bug_feedback.content if bug_feedback else "",
+                filename=self.i_context.filename,
+                summary_log=summary_doc.content if summary_doc else "",
+            )
+        else:
+            prompt = PROMPT_TEMPLATE.format(
+                design=coding_context.design_doc.content if coding_context.design_doc else "",
+                task=coding_context.task_doc.content if coding_context.task_doc else "",
+                code=code_context,
+                logs=logs,
+                feedback=bug_feedback.content if bug_feedback else "",
+                filename=self.i_context.filename,
+                summary_log=summary_doc.content if summary_doc else "",
+            )
         logger.info(f"Writing {coding_context.filename}..")
         code = await self.write_code(prompt)
         if not coding_context.code_doc:
@@ -127,20 +154,66 @@ class WriteCode(Action):
         return coding_context
 
     @staticmethod
-    async def get_codes(task_doc, exclude, project_repo) -> str:
+    async def get_codes(task_doc: Document, exclude: str, project_repo: ProjectRepo, use_inc: bool = False) -> str:
+        """
+        Get codes for generating the exclude file in various scenarios.
+
+        Attributes:
+            task_doc (Document): Document object of the task file.
+            exclude (str): The file to be generated. Specifies the filename to be excluded from the code snippets.
+            project_repo (ProjectRepo): ProjectRepo object of the project.
+            use_inc (bool): Indicates whether the scenario involves incremental development. Defaults to False.
+
+        Returns:
+            str: Codes for generating the exclude file.
+        """
         if not task_doc:
             return ""
         if not task_doc.content:
             task_doc = project_repo.docs.task.get(filename=task_doc.filename)
         m = json.loads(task_doc.content)
-        code_filenames = m.get("Task list", [])
+        code_filenames = m.get(TASK_LIST.key, []) if use_inc else m.get(REFINED_TASK_LIST.key, [])
         codes = []
         src_file_repo = project_repo.srcs
-        for filename in code_filenames:
-            if filename == exclude:
-                continue
-            doc = await src_file_repo.get(filename=filename)
-            if not doc:
-                continue
-            codes.append(f"----- {filename}\n" + doc.content)
+
+        # Incremental development scenario
+        if use_inc:
+            src_files = src_file_repo.all_files
+            # Get the old workspace contained the old codes and old workspace are created in previous CodePlanAndChange
+            old_file_repo = project_repo.git_repo.new_file_repository(relative_path=project_repo.old_workspace)
+            old_files = old_file_repo.all_files
+            # Get the union of the files in the src and old workspaces
+            union_files_list = list(set(src_files) | set(old_files))
+            for filename in union_files_list:
+                # Exclude the current file from the all code snippets
+                if filename == exclude:
+                    # If the file is in the old workspace, use the old code
+                    # Exclude unnecessary code to maintain a clean and focused main.py file, ensuring only relevant and
+                    # essential functionality is included for the project’s requirements
+                    if filename in old_files and filename != "main.py":
+                        # Use old code
+                        doc = await old_file_repo.get(filename=filename)
+                    # If the file is in the src workspace, skip it
+                    else:
+                        continue
+                    codes.insert(0, f"-----Now, {filename} to be rewritten\n```{doc.content}```\n=====")
+                # The code snippets are generated from the src workspace
+                else:
+                    doc = await src_file_repo.get(filename=filename)
+                    # If the file does not exist in the src workspace, skip it
+                    if not doc:
+                        continue
+                    codes.append(f"----- {filename}\n```{doc.content}```")
+
+        # Normal scenario
+        else:
+            for filename in code_filenames:
+                # Exclude the current file to get the code snippets for generating the current file
+                if filename == exclude:
+                    continue
+                doc = await src_file_repo.get(filename=filename)
+                if not doc:
+                    continue
+                codes.append(f"----- {filename}\n```{doc.content}```")
+
         return "\n".join(codes)

metagpt/actions/write_code_plan_and_change_an.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+@Time    : 2023/12/26
+@Author  : mannaandpoem
+@File    : write_code_plan_and_change_an.py
+"""
+import os
+
+from pydantic import Field
+
+from metagpt.actions.action import Action
+from metagpt.actions.action_node import ActionNode
+from metagpt.schema import CodePlanAndChangeContext
+
+CODE_PLAN_AND_CHANGE = ActionNode(
+    key="Code Plan And Change",
+    expected_type=str,
+    instruction="Developing comprehensive and step-by-step incremental development plan, and write Incremental "
+    "Change by making a code draft that how to implement incremental development including detailed steps based on the "
+    "context. Note: Track incremental changes using mark of '+' or '-' for add/modify/delete code, and conforms to the "
+    "output format of git diff",
+    example="""
+1. Plan for calculator.py: Enhance the functionality of `calculator.py` by extending it to incorporate methods for subtraction, multiplication, and division. Additionally, implement robust error handling for the division operation to mitigate potential issues related to division by zero. 
+```python
+class Calculator:
+         self.result = number1 + number2
+         return self.result
+
+-    def sub(self, number1, number2) -> float:
++    def subtract(self, number1: float, number2: float) -> float:
++        '''
++        Subtracts the second number from the first and returns the result.
++
++        Args:
++            number1 (float): The number to be subtracted from.
++            number2 (float): The number to subtract.
++
++        Returns:
++            float: The difference of number1 and number2.
++        '''
++        self.result = number1 - number2
++        return self.result
++
+    def multiply(self, number1: float, number2: float) -> float:
+-        pass
++        '''
++        Multiplies two numbers and returns the result.
++
++        Args:
++            number1 (float): The first number to multiply.
++            number2 (float): The second number to multiply.
++
++        Returns:
++            float: The product of number1 and number2.
++        '''
++        self.result = number1 * number2
++        return self.result
++
+    def divide(self, number1: float, number2: float) -> float:
+-        pass
++        '''
++            ValueError: If the second number is zero.
++        '''
++        if number2 == 0:
++            raise ValueError('Cannot divide by zero')
++        self.result = number1 / number2
++        return self.result
++
+-    def reset_result(self):
++    def clear(self):
++        if self.result != 0.0:
++            print("Result is not zero, clearing...")
++        else:
++            print("Result is already zero, no need to clear.")
++
+         self.result = 0.0
+```
+
+2. Plan for main.py: Integrate new API endpoints for subtraction, multiplication, and division into the existing codebase of `main.py`. Then, ensure seamless integration with the overall application architecture and maintain consistency with coding standards.
+```python
+def add_numbers():
+     result = calculator.add_numbers(num1, num2)
+     return jsonify({'result': result}), 200
+
+-# TODO: Implement subtraction, multiplication, and division operations
++@app.route('/subtract_numbers', methods=['POST'])
++def subtract_numbers():
++    data = request.get_json()
++    num1 = data.get('num1', 0)
++    num2 = data.get('num2', 0)
++    result = calculator.subtract_numbers(num1, num2)
++    return jsonify({'result': result}), 200
++
++@app.route('/multiply_numbers', methods=['POST'])
++def multiply_numbers():
++    data = request.get_json()
++    num1 = data.get('num1', 0)
++    num2 = data.get('num2', 0)
++    try:
++        result = calculator.divide_numbers(num1, num2)
++    except ValueError as e:
++        return jsonify({'error': str(e)}), 400
++    return jsonify({'result': result}), 200
++
+ if __name__ == '__main__':
+     app.run()
+```""",
+)
+
+CODE_PLAN_AND_CHANGE_CONTEXT = """
+## User New Requirements
+{requirement}
+
+## PRD
+{prd}
+
+## Design
+{design}
+
+## Task
+{task}
+
+## Legacy Code
+{code}
+"""
+
+REFINED_TEMPLATE = """
+NOTICE
+Role: You are a professional engineer; The main goal is to complete incremental development by combining legacy code and plan and Incremental Change, ensuring the integration of new features.
+
+# Context
+## User New Requirements
+{user_requirement}
+
+## Code Plan And Change
+{code_plan_and_change}
+
+## Design
+{design}
+
+## Task
+{task}
+
+## Legacy Code
+```Code
+{code}
+```
+
+## Debug logs
+```text
+{logs}
+
+{summary_log}
+```
+
+## Bug Feedback logs
+```text
+{feedback}
+```
+
+# Format example
+## Code: {filename}
+```python
+## {filename}
+...
+```
+
+# Instruction: Based on the context, follow "Format example", write or rewrite code.
+## Write/Rewrite Code: Only write one file {filename}, write or rewrite complete code using triple quotes based on the following attentions and context.
+1. Only One file: do your best to implement THIS ONLY ONE FILE.
+2. COMPLETE CODE: Your code will be part of the entire project, so please implement complete, reliable, reusable code snippets.
+3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. AVOID circular import.
+4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. Do not use public member functions that do not exist in your design.
+5. Follow Code Plan And Change: If there is any Incremental Change that is marked by the git diff format using '+' and '-' for add/modify/delete code, or Legacy Code files contain "{filename} to be rewritten", you must merge it into the code file according to the plan. 
+6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE.
+7. Before using a external variable/module, make sure you import it first.
+8. Write out EVERY CODE DETAIL, DON'T LEAVE TODO.
+9. Attention: Retain details that are not related to incremental development but are important for maintaining the consistency and clarity of the old code.
+"""
+
+WRITE_CODE_PLAN_AND_CHANGE_NODE = ActionNode.from_children("WriteCodePlanAndChange", [CODE_PLAN_AND_CHANGE])
+
+
+class WriteCodePlanAndChange(Action):
+    name: str = "WriteCodePlanAndChange"
+    i_context: CodePlanAndChangeContext = Field(default_factory=CodePlanAndChangeContext)
+
+    async def run(self, *args, **kwargs):
+        self.llm.system_prompt = "You are a professional software engineer, your primary responsibility is to "
+        "meticulously craft comprehensive incremental development plan and deliver detailed incremental change"
+        prd_doc = await self.repo.docs.prd.get(filename=self.i_context.prd_filename)
+        design_doc = await self.repo.docs.system_design.get(filename=self.i_context.design_filename)
+        task_doc = await self.repo.docs.task.get(filename=self.i_context.task_filename)
+        code_text = await self.get_old_codes()
+        context = CODE_PLAN_AND_CHANGE_CONTEXT.format(
+            requirement=self.i_context.requirement,
+            prd=prd_doc.content,
+            design=design_doc.content,
+            task=task_doc.content,
+            code=code_text,
+        )
+        return await WRITE_CODE_PLAN_AND_CHANGE_NODE.fill(context=context, llm=self.llm, schema="json")
+
+    async def get_old_codes(self) -> str:
+        self.repo.old_workspace = self.repo.git_repo.workdir / os.path.basename(self.config.project_path)
+        old_file_repo = self.repo.git_repo.new_file_repository(relative_path=self.repo.old_workspace)
+        old_codes = await old_file_repo.get_all()
+        codes = [f"----- {code.filename}\n```{code.content}```" for code in old_codes]
+        return "\n".join(codes)

metagpt/actions/write_code_review.py

@@ -13,6 +13,7 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions import WriteCode
 from metagpt.actions.action import Action
+from metagpt.const import CODE_PLAN_AND_CHANGE_FILENAME, REQUIREMENT_FILENAME
 from metagpt.logs import logger
 from metagpt.schema import CodingContext
 from metagpt.utils.common import CodeParser
@@ -137,21 +138,38 @@ class WriteCodeReview(Action):
     async def run(self, *args, **kwargs) -> CodingContext:
         iterative_code = self.i_context.code_doc.content
         k = self.context.config.code_review_k_times or 1
+
         for i in range(k):
             format_example = FORMAT_EXAMPLE.format(filename=self.i_context.code_doc.filename)
             task_content = self.i_context.task_doc.content if self.i_context.task_doc else ""
             code_context = await WriteCode.get_codes(
                 self.i_context.task_doc,
                 exclude=self.i_context.filename,
-                project_repo=self.project_repo.with_src_path(self.context.src_workspace),
-            )
-            context = "\n".join(
-                [
-                    "## System Design\n" + str(self.i_context.design_doc) + "\n",
-                    "## Tasks\n" + task_content + "\n",
-                    "## Code Files\n" + code_context + "\n",
-                ]
+                project_repo=self.repo.with_src_path(self.context.src_workspace),
+                use_inc=self.config.inc,
             )
+
+            if not self.config.inc:
+                context = "\n".join(
+                    [
+                        "## System Design\n" + str(self.i_context.design_doc) + "\n",
+                        "## Task\n" + task_content + "\n",
+                        "## Code Files\n" + code_context + "\n",
+                    ]
+                )
+            else:
+                requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME)
+                code_plan_and_change_doc = await self.repo.get(filename=CODE_PLAN_AND_CHANGE_FILENAME)
+                context = "\n".join(
+                    [
+                        "## User New Requirements\n" + str(requirement_doc) + "\n",
+                        "## Code Plan And Change\n" + str(code_plan_and_change_doc) + "\n",
+                        "## System Design\n" + str(self.i_context.design_doc) + "\n",
+                        "## Task\n" + task_content + "\n",
+                        "## Code Files\n" + code_context + "\n",
+                    ]
+                )
+
             context_prompt = PROMPT_TEMPLATE.format(
                 context=context,
                 code=iterative_code,
@@ -160,9 +178,11 @@ class WriteCodeReview(Action):
             cr_prompt = EXAMPLE_AND_INSTRUCTION.format(
                 format_example=format_example,
             )
+            len1 = len(iterative_code) if iterative_code else 0
+            len2 = len(self.i_context.code_doc.content) if self.i_context.code_doc.content else 0
             logger.info(
-                f"Code review and rewrite {self.i_context.code_doc.filename}: {i + 1}/{k} | {len(iterative_code)=}, "
-                f"{len(self.i_context.code_doc.content)=}"
+                f"Code review and rewrite {self.i_context.code_doc.filename}: {i + 1}/{k} | len(iterative_code)={len1}, "
+                f"len(self.i_context.code_doc.content)={len2}"
             )
             result, rewrited_code = await self.write_code_review_and_rewrite(
                 context_prompt, cr_prompt, self.i_context.code_doc.filename

metagpt/actions/write_docstring.py

@@ -16,7 +16,7 @@ Options:
                                 Default: 'google'
 
 Example:
-    python3 -m metagpt.actions.write_docstring ./metagpt/startup.py --overwrite False --style=numpy
+    python3 -m metagpt.actions.write_docstring ./metagpt/software_company.py --overwrite False --style=numpy
 
 This script uses the 'fire' library to create a command-line interface. It generates docstrings for the given Python code using
 the specified docstring style and adds them to the code.

metagpt/actions/write_prd.py

@@ -15,13 +15,14 @@ from __future__ import annotations
 
 import json
 from pathlib import Path
-from typing import Optional
 
 from metagpt.actions import Action, ActionOutput
 from metagpt.actions.action_node import ActionNode
 from metagpt.actions.fix_bug import FixBug
 from metagpt.actions.write_prd_an import (
+    COMPETITIVE_QUADRANT_CHART,
     PROJECT_NAME,
+    REFINED_PRD_NODE,
     WP_IS_RELATIVE_NODE,
     WP_ISSUE_TYPE_NODE,
     WRITE_PRD_NODE,
@@ -58,97 +59,107 @@ NEW_REQ_TEMPLATE = """
 
 
 class WritePRD(Action):
-    name: str = "WritePRD"
-    content: Optional[str] = None
+    """WritePRD deal with the following situations:
+    1. Bugfix: If the requirement is a bugfix, the bugfix document will be generated.
+    2. New requirement: If the requirement is a new requirement, the PRD document will be generated.
+    3. Requirement update: If the requirement is an update, the PRD document will be updated.
+    """
 
     async def run(self, with_messages, *args, **kwargs) -> ActionOutput | Message:
-        # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are
-        # related to the PRD. If they are related, rewrite the PRD.
-        requirement_doc = await self.project_repo.docs.get(filename=REQUIREMENT_FILENAME)
-        if requirement_doc and await self._is_bugfix(requirement_doc.content):
-            await self.project_repo.docs.save(filename=BUGFIX_FILENAME, content=requirement_doc.content)
-            await self.project_repo.docs.save(filename=REQUIREMENT_FILENAME, content="")
-            bug_fix = BugFixContext(filename=BUGFIX_FILENAME)
-            return Message(
-                content=bug_fix.model_dump_json(),
-                instruct_content=bug_fix,
-                role="",
-                cause_by=FixBug,
-                sent_from=self,
-                send_to="Alex",  # the name of Engineer
-            )
+        """Run the action."""
+        req: Document = await self.repo.requirement
+        docs: list[Document] = await self.repo.docs.prd.get_all()
+        if not req:
+            raise FileNotFoundError("No requirement document found.")
+
+        if await self._is_bugfix(req.content):
+            logger.info(f"Bugfix detected: {req.content}")
+            return await self._handle_bugfix(req)
+        # remove bugfix file from last round in case of conflict
+        await self.repo.docs.delete(filename=BUGFIX_FILENAME)
+
+        # if requirement is related to other documents, update them, otherwise create a new one
+        if related_docs := await self.get_related_docs(req, docs):
+            logger.info(f"Requirement update detected: {req.content}")
+            return await self._handle_requirement_update(req, related_docs)
         else:
-            await self.project_repo.docs.delete(filename=BUGFIX_FILENAME)
+            logger.info(f"New requirement detected: {req.content}")
+            return await self._handle_new_requirement(req)
 
-        prd_docs = await self.project_repo.docs.prd.get_all()
-        change_files = Documents()
-        for prd_doc in prd_docs:
-            prd_doc = await self._update_prd(requirement_doc=requirement_doc, prd_doc=prd_doc, *args, **kwargs)
-            if not prd_doc:
-                continue
-            change_files.docs[prd_doc.filename] = prd_doc
-            logger.info(f"rewrite prd: {prd_doc.filename}")
-        # If there is no existing PRD, generate one using 'docs/requirement.txt'.
-        if not change_files.docs:
-            prd_doc = await self._update_prd(requirement_doc=requirement_doc, *args, **kwargs)
-            if prd_doc:
-                change_files.docs[prd_doc.filename] = prd_doc
-                logger.debug(f"new prd: {prd_doc.filename}")
-        # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the
-        # 'publish' message to transition the workflow to the next stage. This design allows room for global
-        # optimization in subsequent steps.
-        return ActionOutput(content=change_files.model_dump_json(), instruct_content=change_files)
+    async def _handle_bugfix(self, req: Document) -> Message:
+        # ... bugfix logic ...
+        await self.repo.docs.save(filename=BUGFIX_FILENAME, content=req.content)
+        await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content="")
+        bug_fix = BugFixContext(filename=BUGFIX_FILENAME)
+        return Message(
+            content=bug_fix.model_dump_json(),
+            instruct_content=bug_fix,
+            role="",
+            cause_by=FixBug,
+            sent_from=self,
+            send_to="Alex",  # the name of Engineer
+        )
 
-    async def _run_new_requirement(self, requirements) -> ActionOutput:
+    async def _handle_new_requirement(self, req: Document) -> ActionOutput:
+        """handle new requirement"""
         project_name = self.project_name
-        context = CONTEXT_TEMPLATE.format(requirements=requirements, project_name=project_name)
+        context = CONTEXT_TEMPLATE.format(requirements=req, project_name=project_name)
         exclude = [PROJECT_NAME.key] if project_name else []
         node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, exclude=exclude)  # schema=schema
         await self._rename_workspace(node)
-        return node
+        new_prd_doc = await self.repo.docs.prd.save(
+            filename=FileRepository.new_filename() + ".json", content=node.instruct_content.model_dump_json()
+        )
+        await self._save_competitive_analysis(new_prd_doc)
+        await self.repo.resources.prd.save_pdf(doc=new_prd_doc)
+        return Documents.from_iterable(documents=[new_prd_doc]).to_action_output()
 
-    async def _is_relative(self, new_requirement_doc, old_prd_doc) -> bool:
-        context = NEW_REQ_TEMPLATE.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content)
+    async def _handle_requirement_update(self, req: Document, related_docs: list[Document]) -> ActionOutput:
+        # ... requirement update logic ...
+        for doc in related_docs:
+            await self._update_prd(req, doc)
+        return Documents.from_iterable(documents=related_docs).to_action_output()
+
+    async def _is_bugfix(self, context: str) -> bool:
+        if not self.repo.code_files_exists():
+            return False
+        node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm)
+        return node.get("issue_type") == "BUG"
+
+    async def get_related_docs(self, req: Document, docs: list[Document]) -> list[Document]:
+        """get the related documents"""
+        # refine: use gather to speed up
+        return [i for i in docs if await self._is_related(req, i)]
+
+    async def _is_related(self, req: Document, old_prd: Document) -> bool:
+        context = NEW_REQ_TEMPLATE.format(old_prd=old_prd.content, requirements=req.content)
         node = await WP_IS_RELATIVE_NODE.fill(context, self.llm)
         return node.get("is_relative") == "YES"
 
-    async def _merge(self, new_requirement_doc, prd_doc) -> Document:
+    async def _merge(self, req: Document, related_doc: Document) -> Document:
         if not self.project_name:
             self.project_name = Path(self.project_path).name
-        prompt = NEW_REQ_TEMPLATE.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content)
-        node = await WRITE_PRD_NODE.fill(context=prompt, llm=self.llm, schema=self.prompt_schema)
-        prd_doc.content = node.instruct_content.model_dump_json()
+        prompt = NEW_REQ_TEMPLATE.format(requirements=req.content, old_prd=related_doc.content)
+        node = await REFINED_PRD_NODE.fill(context=prompt, llm=self.llm, schema=self.prompt_schema)
+        related_doc.content = node.instruct_content.model_dump_json()
         await self._rename_workspace(node)
-        return prd_doc
+        return related_doc
 
-    async def _update_prd(self, requirement_doc, prd_doc=None, *args, **kwargs) -> Document | None:
-        if not prd_doc:
-            prd = await self._run_new_requirement(
-                requirements=[requirement_doc.content if requirement_doc else ""], *args, **kwargs
-            )
-            new_prd_doc = await self.project_repo.docs.prd.save(
-                filename=FileRepository.new_filename() + ".json", content=prd.instruct_content.model_dump_json()
-            )
-        elif await self._is_relative(requirement_doc, prd_doc):
-            new_prd_doc = await self._merge(requirement_doc, prd_doc)
-            self.project_repo.docs.prd.save_doc(doc=new_prd_doc)
-        else:
-            return None
+    async def _update_prd(self, req: Document, prd_doc: Document) -> Document:
+        new_prd_doc: Document = await self._merge(req, prd_doc)
+        await self.repo.docs.prd.save_doc(doc=new_prd_doc)
         await self._save_competitive_analysis(new_prd_doc)
-        await self.project_repo.resources.prd.save_pdf(doc=new_prd_doc)
+        await self.repo.resources.prd.save_pdf(doc=new_prd_doc)
         return new_prd_doc
 
-    async def _save_competitive_analysis(self, prd_doc):
+    async def _save_competitive_analysis(self, prd_doc: Document):
         m = json.loads(prd_doc.content)
-        quadrant_chart = m.get("Competitive Quadrant Chart")
+        quadrant_chart = m.get(COMPETITIVE_QUADRANT_CHART.key)
         if not quadrant_chart:
             return
-        pathname = (
-            self.project_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("")
-        )
-        if not pathname.parent.exists():
-            pathname.parent.mkdir(parents=True, exist_ok=True)
-        await mermaid_to_file(self.config.mermaid_engine, quadrant_chart, pathname)
+        pathname = self.repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO / Path(prd_doc.filename).stem
+        pathname.parent.mkdir(parents=True, exist_ok=True)
+        await mermaid_to_file(self.config.mermaid.engine, quadrant_chart, pathname)
 
     async def _rename_workspace(self, prd):
         if not self.project_name:
@@ -158,15 +169,4 @@ class WritePRD(Action):
                 ws_name = CodeParser.parse_str(block="Project Name", text=prd)
             if ws_name:
                 self.project_name = ws_name
-        self.project_repo.git_repo.rename_root(self.project_name)
-
-    async def _is_bugfix(self, context) -> bool:
-        git_workdir = self.project_repo.git_repo.workdir
-        src_workdir = git_workdir / git_workdir.name
-        if not src_workdir.exists():
-            return False
-        code_files = self.project_repo.with_src_path(path=git_workdir / git_workdir.name).srcs.all_files
-        if not code_files:
-            return False
-        node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm)
-        return node.get("issue_type") == "BUG"
+        self.repo.git_repo.rename_root(self.project_name)

metagpt/actions/write_prd_an.py

@@ -30,6 +30,13 @@ ORIGINAL_REQUIREMENTS = ActionNode(
     example="Create a 2048 game",
 )
 
+REFINED_REQUIREMENTS = ActionNode(
+    key="Refined Requirements",
+    expected_type=str,
+    instruction="Place the New user's original requirements here.",
+    example="Create a 2048 game with a new feature that ...",
+)
+
 PROJECT_NAME = ActionNode(
     key="Project Name",
     expected_type=str,
@@ -45,6 +52,18 @@ PRODUCT_GOALS = ActionNode(
     example=["Create an engaging user experience", "Improve accessibility, be responsive", "More beautiful UI"],
 )
 
+REFINED_PRODUCT_GOALS = ActionNode(
+    key="Refined Product Goals",
+    expected_type=List[str],
+    instruction="Update and expand the original product goals to reflect the evolving needs due to incremental "
+    "development.Ensure that the refined goals align with the current project direction and contribute to its success.",
+    example=[
+        "Enhance user engagement through new features",
+        "Optimize performance for scalability",
+        "Integrate innovative UI enhancements",
+    ],
+)
+
 USER_STORIES = ActionNode(
     key="User Stories",
     expected_type=List[str],
@@ -58,6 +77,20 @@ USER_STORIES = ActionNode(
     ],
 )
 
+REFINED_USER_STORIES = ActionNode(
+    key="Refined User Stories",
+    expected_type=List[str],
+    instruction="Update and expand the original scenario-based user stories to reflect the evolving needs due to "
+    "incremental development. Ensure that the refined user stories capture incremental features and improvements. ",
+    example=[
+        "As a player, I want to choose difficulty levels to challenge my skills",
+        "As a player, I want a visually appealing score display after each game for a better gaming experience",
+        "As a player, I want a convenient restart button displayed when I lose to quickly start a new game",
+        "As a player, I want an enhanced and aesthetically pleasing UI to elevate the overall gaming experience",
+        "As a player, I want the ability to play the game seamlessly on my mobile phone for on-the-go entertainment",
+    ],
+)
+
 COMPETITIVE_ANALYSIS = ActionNode(
     key="Competitive Analysis",
     expected_type=List[str],
@@ -97,6 +130,15 @@ REQUIREMENT_ANALYSIS = ActionNode(
     example="",
 )
 
+REFINED_REQUIREMENT_ANALYSIS = ActionNode(
+    key="Refined Requirement Analysis",
+    expected_type=List[str],
+    instruction="Review and refine the existing requirement analysis to align with the evolving needs of the project "
+    "due to incremental development. Ensure the analysis comprehensively covers the new features and enhancements "
+    "required for the refined project scope.",
+    example=["Require add/update/modify ..."],
+)
+
 REQUIREMENT_POOL = ActionNode(
     key="Requirement Pool",
     expected_type=List[List[str]],
@@ -104,6 +146,14 @@ REQUIREMENT_POOL = ActionNode(
     example=[["P0", "The main code ..."], ["P0", "The game algorithm ..."]],
 )
 
+REFINED_REQUIREMENT_POOL = ActionNode(
+    key="Refined Requirement Pool",
+    expected_type=List[List[str]],
+    instruction="List down the top 5 to 7 requirements with their priority (P0, P1, P2). "
+    "Cover both legacy content and incremental content. Retain content unrelated to incremental development",
+    example=[["P0", "The main code ..."], ["P0", "The game algorithm ..."]],
+)
+
 UI_DESIGN_DRAFT = ActionNode(
     key="UI Design draft",
     expected_type=str,
@@ -152,6 +202,22 @@ NODES = [
     ANYTHING_UNCLEAR,
 ]
 
+REFINED_NODES = [
+    LANGUAGE,
+    PROGRAMMING_LANGUAGE,
+    REFINED_REQUIREMENTS,
+    PROJECT_NAME,
+    REFINED_PRODUCT_GOALS,
+    REFINED_USER_STORIES,
+    COMPETITIVE_ANALYSIS,
+    COMPETITIVE_QUADRANT_CHART,
+    REFINED_REQUIREMENT_ANALYSIS,
+    REFINED_REQUIREMENT_POOL,
+    UI_DESIGN_DRAFT,
+    ANYTHING_UNCLEAR,
+]
+
 WRITE_PRD_NODE = ActionNode.from_children("WritePRD", NODES)
+REFINED_PRD_NODE = ActionNode.from_children("RefinedPRD", REFINED_NODES)
 WP_ISSUE_TYPE_NODE = ActionNode.from_children("WP_ISSUE_TYPE", [ISSUE_TYPE, REASON])
 WP_IS_RELATIVE_NODE = ActionNode.from_children("WP_IS_RELATIVE", [IS_RELATIVE, REASON])

metagpt/config2.py

@@ -38,6 +38,7 @@ class CLIParams(BaseModel):
         if self.project_path:
             self.inc = True
             self.project_name = self.project_name or Path(self.project_path).name
+        return self
 
 
 class Config(CLIParams, YamlModel):
@@ -50,7 +51,7 @@ class Config(CLIParams, YamlModel):
     proxy: str = ""
 
     # Tool Parameters
-    search: Optional[SearchConfig] = None
+    search: SearchConfig = SearchConfig()
     browser: BrowserConfig = BrowserConfig()
     mermaid: MermaidConfig = MermaidConfig()
 
@@ -66,25 +67,22 @@ class Config(CLIParams, YamlModel):
     code_review_k_times: int = 2
 
     # Will be removed in the future
-    llm_for_researcher_summary: str = "gpt3"
-    llm_for_researcher_report: str = "gpt3"
-    METAGPT_TEXT_TO_IMAGE_MODEL_URL: str = ""
+    metagpt_tti_url: str = ""
     language: str = "English"
     redis_key: str = "placeholder"
-    mmdc: str = "mmdc"
-    puppeteer_config: str = ""
-    pyppeteer_executable_path: str = ""
-    IFLYTEK_APP_ID: str = ""
-    IFLYTEK_API_SECRET: str = ""
-    IFLYTEK_API_KEY: str = ""
-    AZURE_TTS_SUBSCRIPTION_KEY: str = ""
-    AZURE_TTS_REGION: str = ""
-    mermaid_engine: str = "nodejs"
+    iflytek_app_id: str = ""
+    iflytek_api_secret: str = ""
+    iflytek_api_key: str = ""
+    azure_tts_subscription_key: str = ""
+    azure_tts_region: str = ""
 
     @classmethod
     def from_home(cls, path):
-        """Load config from ~/.metagpt/config.yaml"""
-        return Config.from_yaml_file(CONFIG_ROOT / path)
+        """Load config from ~/.metagpt/config2.yaml"""
+        pathname = CONFIG_ROOT / path
+        if not pathname.exists():
+            return None
+        return Config.from_yaml_file(pathname)
 
     @classmethod
     def default(cls):

metagpt/configs/browser_config.py

@@ -15,6 +15,6 @@ class BrowserConfig(YamlModel):
     """Config for Browser"""
 
     engine: WebBrowserEngineType = WebBrowserEngineType.PLAYWRIGHT
-    browser: Literal["chrome", "firefox", "edge", "ie"] = "chrome"
-    driver: Literal["chromium", "firefox", "webkit"] = "chromium"
-    path: str = ""
+    browser_type: Literal["chromium", "firefox", "webkit", "chrome", "firefox", "edge", "ie"] = "chromium"
+    """If the engine is Playwright, the value should be one of "chromium", "firefox", or "webkit". If it is Selenium, the value
+    should be either "chrome", "firefox", "edge", or "ie"."""

metagpt/configs/llm_config.py

@@ -74,5 +74,5 @@ class LLMConfig(YamlModel):
     @classmethod
     def check_llm_key(cls, v):
         if v in ["", None, "YOUR_API_KEY"]:
-            raise ValueError("Please set your API key in config.yaml")
+            raise ValueError("Please set your API key in config2.yaml")
         return v

metagpt/configs/mermaid_config.py

@@ -14,5 +14,6 @@ class MermaidConfig(YamlModel):
     """Config for Mermaid"""
 
     engine: Literal["nodejs", "ink", "playwright", "pyppeteer"] = "nodejs"
-    path: str = ""
-    puppeteer_config: str = ""  # Only for nodejs engine
+    path: str = "mmdc"  # mmdc
+    puppeteer_config: str = ""
+    pyppeteer_path: str = "/usr/bin/google-chrome-stable"

metagpt/configs/search_config.py

@@ -5,6 +5,8 @@
 @Author  : alexanderwu
 @File    : search_config.py
 """
+from typing import Callable, Optional
+
 from metagpt.tools import SearchEngineType
 from metagpt.utils.yaml_model import YamlModel
 
@@ -12,6 +14,7 @@ from metagpt.utils.yaml_model import YamlModel
 class SearchConfig(YamlModel):
     """Config for Search"""
 
-    api_key: str
-    api_type: SearchEngineType = SearchEngineType.SERPAPI_GOOGLE
+    api_type: SearchEngineType = SearchEngineType.DUCK_DUCK_GO
+    api_key: str = ""
     cse_id: str = ""  # for google
+    search_func: Optional[Callable] = None

metagpt/const.py

@@ -67,6 +67,8 @@ TMP = METAGPT_ROOT / "tmp"
 SOURCE_ROOT = METAGPT_ROOT / "metagpt"
 PROMPT_PATH = SOURCE_ROOT / "prompts"
 SKILL_DIRECTORY = SOURCE_ROOT / "skills"
+TOOL_SCHEMA_PATH = METAGPT_ROOT / "metagpt/tools/schemas"
+TOOL_LIBS_PATH = METAGPT_ROOT / "metagpt/tools/libs"
 
 # REAL CONSTS
 
@@ -82,17 +84,20 @@ MESSAGE_ROUTE_TO_NONE = "<none>"
 REQUIREMENT_FILENAME = "requirement.txt"
 BUGFIX_FILENAME = "bugfix.txt"
 PACKAGE_REQUIREMENTS_FILENAME = "requirements.txt"
+CODE_PLAN_AND_CHANGE_FILENAME = "code_plan_and_change.json"
 
 DOCS_FILE_REPO = "docs"
 PRDS_FILE_REPO = "docs/prd"
 SYSTEM_DESIGN_FILE_REPO = "docs/system_design"
 TASK_FILE_REPO = "docs/task"
+CODE_PLAN_AND_CHANGE_FILE_REPO = "docs/code_plan_and_change"
 COMPETITIVE_ANALYSIS_FILE_REPO = "resources/competitive_analysis"
 DATA_API_DESIGN_FILE_REPO = "resources/data_api_design"
 SEQ_FLOW_FILE_REPO = "resources/seq_flow"
 SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design"
 PRD_PDF_FILE_REPO = "resources/prd"
 TASK_PDF_FILE_REPO = "resources/api_spec_and_task"
+CODE_PLAN_AND_CHANGE_PDF_FILE_REPO = "resources/code_plan_and_change"
 TEST_CODES_FILE_REPO = "tests"
 TEST_OUTPUTS_FILE_REPO = "test_outputs"
 CODE_SUMMARIES_FILE_REPO = "docs/code_summary"

metagpt/context.py

@@ -17,6 +17,7 @@ from metagpt.provider.base_llm import BaseLLM
 from metagpt.provider.llm_provider_registry import create_llm_instance
 from metagpt.utils.cost_manager import CostManager
 from metagpt.utils.git_repository import GitRepository
+from metagpt.utils.project_repo import ProjectRepo
 
 
 class AttrDict(BaseModel):
@@ -58,6 +59,8 @@ class Context(BaseModel):
 
     kwargs: AttrDict = AttrDict()
     config: Config = Config.default()
+
+    repo: Optional[ProjectRepo] = None
     git_repo: Optional[GitRepository] = None
     src_workspace: Optional[Path] = None
     cost_manager: CostManager = CostManager()
@@ -67,8 +70,8 @@ class Context(BaseModel):
     def new_environ(self):
         """Return a new os.environ object"""
         env = os.environ.copy()
-        i = self.options
-        env.update({k: v for k, v in i.items() if isinstance(v, str)})
+        # i = self.options
+        # env.update({k: v for k, v in i.items() if isinstance(v, str)})
         return env
 
     # def use_llm(self, name: Optional[str] = None, provider: LLMType = LLMType.OPENAI) -> BaseLLM:
@@ -92,7 +95,3 @@ class Context(BaseModel):
         if llm.cost_manager is None:
             llm.cost_manager = self.cost_manager
         return llm
-
-
-# Global context, not in Env
-CONTEXT = Context()

metagpt/context_mixin.py

@@ -7,17 +7,17 @@
 """
 from typing import Optional
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 from metagpt.config2 import Config
-from metagpt.context import CONTEXT, Context
+from metagpt.context import Context
 from metagpt.provider.base_llm import BaseLLM
 
 
 class ContextMixin(BaseModel):
     """Mixin class for context and config"""
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
+    model_config = ConfigDict(arbitrary_types_allowed=True, extra="allow")
 
     # Pydantic has bug on _private_attr when using inheritance, so we use private_* instead
     # - https://github.com/pydantic/pydantic/issues/7142
@@ -32,18 +32,17 @@ class ContextMixin(BaseModel):
     # Env/Role/Action will use this llm as private llm, or use self.context._llm instance
     private_llm: Optional[BaseLLM] = Field(default=None, exclude=True)
 
-    def __init__(
-        self,
-        context: Optional[Context] = CONTEXT,
-        config: Optional[Config] = None,
-        llm: Optional[BaseLLM] = None,
-        **kwargs,
-    ):
-        """Initialize with config"""
-        super().__init__(**kwargs)
-        self.set_context(context)
-        self.set_config(config)
-        self.set_llm(llm)
+    @model_validator(mode="after")
+    def validate_context_mixin_extra(self):
+        self._process_context_mixin_extra()
+        return self
+
+    def _process_context_mixin_extra(self):
+        """Process the extra field"""
+        kwargs = self.model_extra or {}
+        self.set_context(kwargs.pop("context", None))
+        self.set_config(kwargs.pop("config", None))
+        self.set_llm(kwargs.pop("llm", None))
 
     def set(self, k, v, override=False):
         """Set attribute"""
@@ -81,7 +80,7 @@ class ContextMixin(BaseModel):
         """Role context: role context > context"""
         if self.private_context:
             return self.private_context
-        return CONTEXT
+        return Context()
 
     @context.setter
     def context(self, context: Context) -> None:

metagpt/environment/README.md

@@ -0,0 +1,38 @@
+Here is a environment description of MetaGPT env for different situation.  
+For now, the code only define the environment and still some todos like migrate roles/actions to current version.
+
+## Function
+- Define `ExtEnv`(Base Class) which help users to integrate with external environment like games through apis or construct the game logics.
+- Define `Environment`(Base Class) which is the env that MetaGPT directly used. And it includes roles and so on.
+- Define the `EnvAPIRegistry` to mark the read/write apis that `ExtEnv` provide observe/step ability. And then, users can call the particular one to get observation from env or feedback to env.
+
+## Usage
+
+init environment
+```
+android_env = env.create(EnvType.ANDROID)
+
+assistant = Role(name="Bob", profile="android assistant")
+team = Team(investment=10.0, env=android_env, roles=[assistant])
+```
+
+observe & step inside role's actions
+```
+from metagpt.environment.api.env_api import EnvAPIAbstract
+
+# get screenshot from ExtEnv
+screenshot_path: Path = env.observe(
+            EnvAPIAbstract(
+                api_name="get_screenshot", kwargs={"ss_name": f"{round_count}_before", "local_save_dir": task_dir}
+            )
+        )
+
+# do a `tap` action on the screen
+res = env.step(EnvAPIAbstract("system_tap", kwargs={"x": x, "y": y}))
+```
+
+## TODO
+- add android app operation assistant under `examples/android_assistant`
+- migrate roles/actions of werewolf game from old version into current version
+- migrate roles/actions of mincraft game from old version into current version
+- migrate roles/actions of stanford_town game from old version into current version

metagpt/environment/__init__.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :
+
+from metagpt.environment.base_env import Environment
+from metagpt.environment.android_env.android_env import AndroidEnv
+from metagpt.environment.mincraft_env.mincraft_env import MincraftExtEnv
+from metagpt.environment.werewolf_env.werewolf_env import WerewolfEnv
+from metagpt.environment.stanford_town_env.stanford_town_env import StanfordTownEnv
+from metagpt.environment.software_env.software_env import SoftwareEnv
+
+
+__all__ = ["AndroidEnv", "MincraftExtEnv", "WerewolfEnv", "StanfordTownEnv", "SoftwareEnv", "Environment"]

metagpt/environment/android_env/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/environment/android_env/android_env.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : MG Android Env
+
+from pydantic import Field
+
+from metagpt.environment.android_env.android_ext_env import AndroidExtEnv
+from metagpt.environment.base_env import Environment
+
+
+class AndroidEnv(Environment, AndroidExtEnv):
+    rows: int = Field(default=0, description="rows of a grid on the screenshot")
+    cols: int = Field(default=0, description="cols of a grid on the screenshot")

metagpt/environment/android_env/android_ext_env.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : The Android external environment to integrate with Android apps
+
+import subprocess
+from pathlib import Path
+from typing import Any, Optional
+
+from pydantic import Field
+
+from metagpt.environment.android_env.const import ADB_EXEC_FAIL
+from metagpt.environment.base_env import ExtEnv, mark_as_readable, mark_as_writeable
+
+
+class AndroidExtEnv(ExtEnv):
+    device_id: Optional[str] = Field(default=None)
+    screenshot_dir: Optional[Path] = Field(default=None)
+    xml_dir: Optional[Path] = Field(default=None)
+    width: int = Field(default=720, description="device screen width")
+    height: int = Field(default=1080, description="device screen height")
+
+    def __init__(self, **data: Any):
+        super().__init__(**data)
+        if data.get("device_id"):
+            (width, height) = self.device_shape
+            self.width = data.get("width", width)
+            self.height = data.get("height", height)
+
+    @property
+    def adb_prefix_si(self):
+        """adb cmd prefix with `device_id` and `shell input`"""
+        return f"adb -s {self.device_id} shell input "
+
+    @property
+    def adb_prefix_shell(self):
+        """adb cmd prefix with `device_id` and `shell`"""
+        return f"adb -s {self.device_id} shell "
+
+    @property
+    def adb_prefix(self):
+        """adb cmd prefix with `device_id`"""
+        return f"adb -s {self.device_id} "
+
+    def execute_adb_with_cmd(self, adb_cmd: str) -> str:
+        res = subprocess.run(adb_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+        exec_res = ADB_EXEC_FAIL
+        if not res.returncode:
+            exec_res = res.stdout.strip()
+        return exec_res
+
+    @property
+    def device_shape(self) -> tuple[int, int]:
+        adb_cmd = f"{self.adb_prefix_shell} wm size"
+        shape = (0, 0)
+        shape_res = self.execute_adb_with_cmd(adb_cmd)
+        if shape_res != ADB_EXEC_FAIL:
+            shape = tuple(map(int, shape_res.split(": ")[1].split("x")))
+        return shape
+
+    def list_devices(self):
+        adb_cmd = "adb devices"
+        res = self.execute_adb_with_cmd(adb_cmd)
+        devices = []
+        if res != ADB_EXEC_FAIL:
+            devices = res.split("\n")[1:]
+            devices = [device.split()[0] for device in devices]
+        return devices
+
+    @mark_as_readable
+    def get_screenshot(self, ss_name: str, local_save_dir: Path) -> Path:
+        """
+        ss_name: screenshot file name
+        local_save_dir: local dir to store image from virtual machine
+        """
+        assert self.screenshot_dir
+        ss_remote_path = Path(self.screenshot_dir).joinpath(f"{ss_name}.png")
+        ss_cmd = f"{self.adb_prefix_shell} screencap -p {ss_remote_path}"
+        ss_res = self.execute_adb_with_cmd(ss_cmd)
+
+        res = ADB_EXEC_FAIL
+        if ss_res != ADB_EXEC_FAIL:
+            ss_local_path = Path(local_save_dir).joinpath(f"{ss_name}.png")
+            pull_cmd = f"{self.adb_prefix} pull {ss_remote_path} {ss_local_path}"
+            pull_res = self.execute_adb_with_cmd(pull_cmd)
+            if pull_res != ADB_EXEC_FAIL:
+                res = ss_local_path
+        return Path(res)
+
+    @mark_as_readable
+    def get_xml(self, xml_name: str, local_save_dir: Path) -> Path:
+        xml_remote_path = Path(self.xml_dir).joinpath(f"{xml_name}.xml")
+        dump_cmd = f"{self.adb_prefix_shell} uiautomator dump {xml_remote_path}"
+        xml_res = self.execute_adb_with_cmd(dump_cmd)
+
+        res = ADB_EXEC_FAIL
+        if xml_res != ADB_EXEC_FAIL:
+            xml_local_path = Path(local_save_dir).joinpath(f"{xml_name}.xml")
+            pull_cmd = f"{self.adb_prefix} pull {xml_remote_path} {xml_local_path}"
+            pull_res = self.execute_adb_with_cmd(pull_cmd)
+            if pull_res != ADB_EXEC_FAIL:
+                res = xml_local_path
+        return Path(res)
+
+    @mark_as_writeable
+    def system_back(self) -> str:
+        adb_cmd = f"{self.adb_prefix_si} keyevent KEYCODE_BACK"
+        back_res = self.execute_adb_with_cmd(adb_cmd)
+        return back_res
+
+    @mark_as_writeable
+    def system_tap(self, x: int, y: int) -> str:
+        adb_cmd = f"{self.adb_prefix_si} tap {x} {y}"
+        tap_res = self.execute_adb_with_cmd(adb_cmd)
+        return tap_res
+
+    @mark_as_writeable
+    def user_input(self, input_txt: str) -> str:
+        input_txt = input_txt.replace(" ", "%s").replace("'", "")
+        adb_cmd = f"{self.adb_prefix_si} text {input_txt}"
+        input_res = self.execute_adb_with_cmd(adb_cmd)
+        return input_res
+
+    @mark_as_writeable
+    def user_longpress(self, x: int, y: int, duration: int = 500) -> str:
+        adb_cmd = f"{self.adb_prefix_si} swipe {x} {y} {x} {y} {duration}"
+        press_res = self.execute_adb_with_cmd(adb_cmd)
+        return press_res
+
+    @mark_as_writeable
+    def user_swipe(self, x: int, y: int, orient: str = "up", dist: str = "medium", if_quick: bool = False) -> str:
+        dist_unit = int(self.width / 10)
+        if dist == "long":
+            dist_unit *= 3
+        elif dist == "medium":
+            dist_unit *= 2
+
+        if orient == "up":
+            offset = 0, -2 * dist_unit
+        elif orient == "down":
+            offset = 0, 2 * dist_unit
+        elif orient == "left":
+            offset = -1 * dist_unit, 0
+        elif orient == "right":
+            offset = dist_unit, 0
+        else:
+            return ADB_EXEC_FAIL
+
+        duration = 100 if if_quick else 400
+        adb_cmd = f"{self.adb_prefix_si} swipe {x} {y} {x + offset[0]} {y + offset[1]} {duration}"
+        swipe_res = self.execute_adb_with_cmd(adb_cmd)
+        return swipe_res
+
+    @mark_as_writeable
+    def user_swipe_to(self, start: tuple[int, int], end: tuple[int, int], duration: int = 400):
+        adb_cmd = f"{self.adb_prefix_si} swipe {start[0]} {start[1]} {end[0]} {end[1]} {duration}"
+        swipe_res = self.execute_adb_with_cmd(adb_cmd)
+        return swipe_res

metagpt/environment/android_env/const.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :
+
+# For Android Assistant Agent
+ADB_EXEC_FAIL = "FAILED"

metagpt/environment/api/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/environment/api/env_api.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :  the environment api store
+
+from typing import Any, Callable, Union
+
+from pydantic import BaseModel, Field
+
+
+class EnvAPIAbstract(BaseModel):
+    """api/interface summary description"""
+
+    api_name: str = Field(default="", description="the api function name or id")
+    args: set = Field(default={}, description="the api function `args` params")
+    kwargs: dict = Field(default=dict(), description="the api function `kwargs` params")
+
+
+class EnvAPIRegistry(BaseModel):
+    """the registry to store environment w&r api/interface"""
+
+    registry: dict[str, dict[str, Union[dict, Any, str]]] = Field(default=dict(), exclude=True)
+
+    def get(self, api_name: str):
+        if api_name not in self.registry:
+            raise ValueError
+        return self.registry.get(api_name)
+
+    def __getitem__(self, api_name: str) -> Callable:
+        return self.get(api_name)
+
+    def __setitem__(self, api_name: str, func: Callable):
+        self.registry[api_name] = func
+
+    def __len__(self):
+        return len(self.registry)
+
+    def get_apis(self, as_str=True) -> dict[str, dict[str, Union[dict, Any, str]]]:
+        """return func schema without func instance"""
+        apis = dict()
+        for func_name, func_schema in self.registry.items():
+            new_func_schema = dict()
+            for key, value in func_schema.items():
+                if key == "func":
+                    continue
+                new_func_schema[key] = str(value) if as_str else value
+            new_func_schema = new_func_schema
+            apis[func_name] = new_func_schema
+        return apis
+
+
+class WriteAPIRegistry(EnvAPIRegistry):
+    """just as a explicit class name"""
+
+    pass
+
+
+class ReadAPIRegistry(EnvAPIRegistry):
+    """just as a explicit class name"""
+
+    pass

metagpt/environment/base_env.py

@@ -1,29 +1,101 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-"""
-@Time    : 2023/5/11 22:12
-@Author  : alexanderwu
-@File    : environment.py
-@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.2 of RFC 116:
-    1. Remove the functionality of `Environment` class as a public message buffer.
-    2. Standardize the message forwarding behavior of the `Environment` class.
-    3. Add the `is_idle` property.
-@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing
-    functionality is to be consolidated into the `Environment` class.
-"""
+# @Desc   : base env of executing environment
+
 import asyncio
-from typing import Iterable, Set
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Set, Union
 
 from pydantic import BaseModel, ConfigDict, Field, SerializeAsAny, model_validator
 
 from metagpt.context import Context
+from metagpt.environment.api.env_api import (
+    EnvAPIAbstract,
+    ReadAPIRegistry,
+    WriteAPIRegistry,
+)
 from metagpt.logs import logger
-from metagpt.roles.role import Role
 from metagpt.schema import Message
-from metagpt.utils.common import is_send_to
+from metagpt.utils.common import get_function_schema, is_coroutine_func, is_send_to
+
+if TYPE_CHECKING:
+    from metagpt.roles.role import Role  # noqa: F401
 
 
-class Environment(BaseModel):
+class EnvType(Enum):
+    ANDROID = "Android"
+    GYM = "Gym"
+    WEREWOLF = "Werewolf"
+    MINCRAFT = "Mincraft"
+    STANFORDTOWN = "StanfordTown"
+
+
+env_write_api_registry = WriteAPIRegistry()
+env_read_api_registry = ReadAPIRegistry()
+
+
+def mark_as_readable(func):
+    """mark functionn as a readable one in ExtEnv, it observes something from ExtEnv"""
+    env_read_api_registry[func.__name__] = get_function_schema(func)
+    return func
+
+
+def mark_as_writeable(func):
+    """mark functionn as a writeable one in ExtEnv, it does something to ExtEnv"""
+    env_write_api_registry[func.__name__] = get_function_schema(func)
+    return func
+
+
+class ExtEnv(BaseModel):
+    """External Env to intergate actual game environment"""
+
+    def _check_api_exist(self, rw_api: Optional[str] = None):
+        if not rw_api:
+            raise ValueError(f"{rw_api} not exists")
+
+    def get_all_available_apis(self, mode: str = "read") -> list[Any]:
+        """get available read/write apis definition"""
+        assert mode in ["read", "write"]
+        if mode == "read":
+            return env_read_api_registry.get_apis()
+        else:
+            return env_write_api_registry.get_apis()
+
+    async def observe(self, env_action: Union[str, EnvAPIAbstract]):
+        """get observation from particular api of ExtEnv"""
+        if isinstance(env_action, str):
+            read_api = env_read_api_registry.get(api_name=env_action)["func"]
+            self._check_api_exist(read_api)
+            if is_coroutine_func(read_api):
+                res = await read_api(self)
+            else:
+                res = read_api(self)
+        elif isinstance(env_action, EnvAPIAbstract):
+            read_api = env_read_api_registry.get(api_name=env_action.api_name)["func"]
+            self._check_api_exist(read_api)
+            if is_coroutine_func(read_api):
+                res = await read_api(self, *env_action.args, **env_action.kwargs)
+            else:
+                res = read_api(self, *env_action.args, **env_action.kwargs)
+        return res
+
+    async def step(self, env_action: Union[str, Message, EnvAPIAbstract, list[EnvAPIAbstract]]):
+        """execute through particular api of ExtEnv"""
+        res = None
+        if isinstance(env_action, Message):
+            self.publish_message(env_action)
+        elif isinstance(env_action, EnvAPIAbstract):
+            write_api = env_write_api_registry.get(env_action.api_name)["func"]
+            self._check_api_exist(write_api)
+            if is_coroutine_func(write_api):
+                res = await write_api(self, *env_action.args, **env_action.kwargs)
+            else:
+                res = write_api(self, *env_action.args, **env_action.kwargs)
+
+        return res
+
+
+class Environment(ExtEnv):
     """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到
     Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles
     """
@@ -31,8 +103,8 @@ class Environment(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     desc: str = Field(default="")  # 环境描述
-    roles: dict[str, SerializeAsAny[Role]] = Field(default_factory=dict, validate_default=True)
-    member_addrs: dict[Role, Set] = Field(default_factory=dict, exclude=True)
+    roles: dict[str, SerializeAsAny["Role"]] = Field(default_factory=dict, validate_default=True)
+    member_addrs: Dict["Role", Set] = Field(default_factory=dict, exclude=True)
     history: str = ""  # For debug
     context: Context = Field(default_factory=Context, exclude=True)
 
@@ -41,7 +113,7 @@ class Environment(BaseModel):
         self.add_roles(self.roles.values())
         return self
 
-    def add_role(self, role: Role):
+    def add_role(self, role: "Role"):
         """增加一个在当前环境的角色
         Add a role in the current environment
         """
@@ -49,7 +121,7 @@ class Environment(BaseModel):
         role.set_env(self)
         role.context = self.context
 
-    def add_roles(self, roles: Iterable[Role]):
+    def add_roles(self, roles: Iterable["Role"]):
         """增加一批在当前环境的角色
         Add a batch of characters in the current environment
         """
@@ -95,13 +167,13 @@ class Environment(BaseModel):
             await asyncio.gather(*futures)
             logger.debug(f"is idle: {self.is_idle}")
 
-    def get_roles(self) -> dict[str, Role]:
+    def get_roles(self) -> dict[str, "Role"]:
         """获得环境内的所有角色
         Process all Role runs at once
         """
         return self.roles
 
-    def get_role(self, name: str) -> Role:
+    def get_role(self, name: str) -> "Role":
         """获得环境内的指定角色
         get all the environment roles
         """
@@ -129,3 +201,12 @@ class Environment(BaseModel):
     def archive(self, auto_archive=True):
         if auto_archive and self.context.git_repo:
             self.context.git_repo.archive()
+
+    @classmethod
+    def model_rebuild(cls, **kwargs):
+        from metagpt.roles.role import Role  # noqa: F401
+
+        super().model_rebuild(**kwargs)
+
+
+Environment.model_rebuild()

metagpt/environment/mincraft_env/__init__.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :

metagpt/environment/mincraft_env/const.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   :
+
+from metagpt.const import METAGPT_ROOT
+
+# For Mincraft Game Agent
+MC_CKPT_DIR = METAGPT_ROOT / "data/mincraft/ckpt"
+MC_LOG_DIR = METAGPT_ROOT / "logs"
+MC_DEFAULT_WARMUP = {
+    "context": 15,
+    "biome": 10,
+    "time": 15,
+    "nearby_blocks": 0,
+    "other_blocks": 10,
+    "nearby_entities": 5,
+    "health": 15,
+    "hunger": 15,
+    "position": 0,
+    "equipment": 0,
+    "inventory": 0,
+    "optional_inventory_items": 7,
+    "chests": 0,
+    "completed_tasks": 0,
+    "failed_tasks": 0,
+}
+MC_CURRICULUM_OB = [
+    "context",
+    "biome",
+    "time",
+    "nearby_blocks",
+    "other_blocks",
+    "nearby_entities",
+    "health",
+    "hunger",
+    "position",
+    "equipment",
+    "inventory",
+    "chests",
+    "completed_tasks",
+    "failed_tasks",
+]
+MC_CORE_INVENTORY_ITEMS = r".*_log|.*_planks|stick|crafting_table|furnace"
+r"|cobblestone|dirt|coal|.*_pickaxe|.*_sword|.*_axe",  # curriculum_agent: only show these items in inventory before optional_inventory_items reached in warm up

metagpt/environment/mincraft_env/mincraft_env.py

@@ -0,0 +1,391 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : MG Mincraft Env
+#           refs to `voyager voyager.py`
+
+import json
+import re
+import time
+from typing import Any, Iterable
+
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from pydantic import ConfigDict, Field
+
+from metagpt.config2 import config as CONFIG
+from metagpt.environment.base_env import Environment
+from metagpt.environment.mincraft_env.const import MC_CKPT_DIR
+from metagpt.environment.mincraft_env.mincraft_ext_env import MincraftExtEnv
+from metagpt.logs import logger
+from metagpt.utils.common import load_mc_skills_code, read_json_file, write_json_file
+
+
+class MincraftEnv(Environment, MincraftExtEnv):
+    """MincraftEnv, including shared memory of cache and infomation between roles"""
+
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    event: dict[str, Any] = Field(default_factory=dict)
+    current_task: str = Field(default="Mine 1 wood log")
+    task_execution_time: float = Field(default=float)
+    context: str = Field(default="You can mine one of oak, birch, spruce, jungle, acacia, dark oak, or mangrove logs.")
+    code: str = Field(default="")
+    program_code: str = Field(default="")  # write in skill/code/*.js
+    program_name: str = Field(default="")
+    critique: str = Field(default="")
+    skills: dict = Field(default_factory=dict)  # for skills.json
+    retrieve_skills: list[str] = Field(default_factory=list)
+    event_summary: str = Field(default="")
+
+    qa_cache: dict[str, str] = Field(default_factory=dict)
+    completed_tasks: list[str] = Field(default_factory=list)  # Critique things
+    failed_tasks: list[str] = Field(default_factory=list)
+
+    skill_desp: str = Field(default="")
+
+    chest_memory: dict[str, Any] = Field(default_factory=dict)  # eg: {'(1344, 64, 1381)': 'Unknown'}
+    chest_observation: str = Field(default="")  # eg: "Chests: None\n\n"
+
+    runtime_status: bool = False  # equal to action execution status: success or failed
+
+    vectordb: Chroma = Field(default_factory=Chroma)
+
+    qa_cache_questions_vectordb: Chroma = Field(default_factory=Chroma)
+
+    @property
+    def progress(self):
+        # return len(self.completed_tasks) + 10 # Test only
+        return len(self.completed_tasks)
+
+    @property
+    def programs(self):
+        programs = ""
+        if self.code == "":
+            return programs  # TODO: maybe fix 10054 now, a better way is isolating env.step() like voyager
+        for skill_name, entry in self.skills.items():
+            programs += f"{entry['code']}\n\n"
+        for primitives in load_mc_skills_code():  # TODO add skills_dir
+            programs += f"{primitives}\n\n"
+        return programs
+
+    def set_mc_port(self, mc_port):
+        super().set_mc_port(mc_port)
+        self.set_mc_resume()
+
+    def set_mc_resume(self):
+        self.qa_cache_questions_vectordb = Chroma(
+            collection_name="qa_cache_questions_vectordb",
+            embedding_function=OpenAIEmbeddings(),
+            persist_directory=f"{MC_CKPT_DIR}/curriculum/vectordb",
+        )
+
+        self.vectordb = Chroma(
+            collection_name="skill_vectordb",
+            embedding_function=OpenAIEmbeddings(),
+            persist_directory=f"{MC_CKPT_DIR}/skill/vectordb",
+        )
+
+        if CONFIG.resume:
+            logger.info(f"Loading Action Developer from {MC_CKPT_DIR}/action")
+            self.chest_memory = read_json_file(f"{MC_CKPT_DIR}/action/chest_memory.json")
+
+            logger.info(f"Loading Curriculum Agent from {MC_CKPT_DIR}/curriculum")
+            self.completed_tasks = read_json_file(f"{MC_CKPT_DIR}/curriculum/completed_tasks.json")
+            self.failed_tasks = read_json_file(f"{MC_CKPT_DIR}/curriculum/failed_tasks.json")
+
+            logger.info(f"Loading Skill Manager from {MC_CKPT_DIR}/skill\033[0m")
+            self.skills = read_json_file(f"{MC_CKPT_DIR}/skill/skills.json")
+
+            logger.info(f"Loading Qa Cache from {MC_CKPT_DIR}/curriculum\033[0m")
+            self.qa_cache = read_json_file(f"{MC_CKPT_DIR}/curriculum/qa_cache.json")
+
+            if self.vectordb._collection.count() == 0:
+                logger.info(self.vectordb._collection.count())
+                # Set vdvs for skills & qa_cache
+                skill_desps = [skill["description"] for program_name, skill in self.skills.items()]
+                program_names = [program_name for program_name, skill in self.skills.items()]
+                metadatas = [{"name": program_name} for program_name in program_names]
+                # add vectordb from file
+                self.vectordb.add_texts(
+                    texts=skill_desps,
+                    ids=program_names,
+                    metadatas=metadatas,
+                )
+                self.vectordb.persist()
+
+            logger.info(self.qa_cache_questions_vectordb._collection.count())
+            if self.qa_cache_questions_vectordb._collection.count() == 0:
+                questions = [question for question, answer in self.qa_cache.items()]
+
+                self.qa_cache_questions_vectordb.add_texts(texts=questions)
+
+                self.qa_cache_questions_vectordb.persist()
+
+                logger.info(
+                    f"INIT_CHECK: There are {self.vectordb._collection.count()} skills in vectordb and {len(self.skills)} skills in skills.json."
+                )
+                # Check if Skill Manager's vectordb right using
+                assert self.vectordb._collection.count() == len(self.skills), (
+                    f"Skill Manager's vectordb is not synced with skills.json.\n"
+                    f"There are {self.vectordb._collection.count()} skills in vectordb but {len(self.skills)} skills in skills.json.\n"
+                    f"Did you set resume=False when initializing the manager?\n"
+                    f"You may need to manually delete the vectordb directory for running from scratch."
+                )
+
+                logger.info(
+                    f"INIT_CHECK: There are {self.qa_cache_questions_vectordb._collection.count()} qa_cache in vectordb and {len(self.qa_cache)} questions in qa_cache.json."
+                )
+                assert self.qa_cache_questions_vectordb._collection.count() == len(self.qa_cache), (
+                    f"Curriculum Agent's qa cache question vectordb is not synced with qa_cache.json.\n"
+                    f"There are {self.qa_cache_questions_vectordb._collection.count()} questions in vectordb "
+                    f"but {len(self.qa_cache)} questions in qa_cache.json.\n"
+                    f"Did you set resume=False when initializing the agent?\n"
+                    f"You may need to manually delete the qa cache question vectordb directory for running from scratch.\n"
+                )
+
+    def register_roles(self, roles: Iterable["Minecraft"]):
+        for role in roles:
+            role.set_memory(self)
+
+    def update_event(self, event: dict):
+        if self.event == event:
+            return
+        self.event = event
+        self.update_chest_memory(event)
+        self.update_chest_observation()
+        # self.event_summary = self.summarize_chatlog(event)
+
+    def update_task(self, task: str):
+        self.current_task = task
+
+    def update_context(self, context: str):
+        self.context = context
+
+    def update_program_code(self, program_code: str):
+        self.program_code = program_code
+
+    def update_code(self, code: str):
+        self.code = code  # action_developer.gen_action_code to HERE
+
+    def update_program_name(self, program_name: str):
+        self.program_name = program_name
+
+    def update_critique(self, critique: str):
+        self.critique = critique  # critic_agent.check_task_success to HERE
+
+    def append_skill(self, skill: dict):
+        self.skills[self.program_name] = skill  # skill_manager.retrieve_skills to HERE
+
+    def update_retrieve_skills(self, retrieve_skills: list):
+        self.retrieve_skills = retrieve_skills
+
+    def update_skill_desp(self, skill_desp: str):
+        self.skill_desp = skill_desp
+
+    async def update_qa_cache(self, qa_cache: dict):
+        self.qa_cache = qa_cache
+
+    def update_chest_memory(self, events: dict):
+        """
+        Input: events: Dict
+        Result: self.chest_memory update & save to json
+        """
+        nearbyChests = events[-1][1]["nearbyChests"]
+        for position, chest in nearbyChests.items():
+            if position in self.chest_memory:
+                if isinstance(chest, dict):
+                    self.chest_memory[position] = chest
+                if chest == "Invalid":
+                    logger.info(f"Action Developer removing chest {position}: {chest}")
+                    self.chest_memory.pop(position)
+            else:
+                if chest != "Invalid":
+                    logger.info(f"Action Developer saving chest {position}: {chest}")
+                    self.chest_memory[position] = chest
+
+        write_json_file(f"{MC_CKPT_DIR}/action/chest_memory.json", self.chest_memory)
+
+    def update_chest_observation(self):
+        """
+        update chest_memory to chest_observation.
+        Refer to @ https://github.com/MineDojo/Voyager/blob/main/voyager/agents/action.py
+        """
+
+        chests = []
+        for chest_position, chest in self.chest_memory.items():
+            if isinstance(chest, dict) and len(chest) > 0:
+                chests.append(f"{chest_position}: {chest}")
+        for chest_position, chest in self.chest_memory.items():
+            if isinstance(chest, dict) and len(chest) == 0:
+                chests.append(f"{chest_position}: Empty")
+        for chest_position, chest in self.chest_memory.items():
+            if isinstance(chest, str):
+                assert chest == "Unknown"
+                chests.append(f"{chest_position}: Unknown items inside")
+        assert len(chests) == len(self.chest_memory)
+        if chests:
+            chests = "\n".join(chests)
+            self.chest_observation = f"Chests:\n{chests}\n\n"
+        else:
+            self.chest_observation = "Chests: None\n\n"
+
+    def summarize_chatlog(self, events):
+        def filter_item(message: str):
+            craft_pattern = r"I cannot make \w+ because I need: (.*)"
+            craft_pattern2 = r"I cannot make \w+ because there is no crafting table nearby"
+            mine_pattern = r"I need at least a (.*) to mine \w+!"
+            if re.match(craft_pattern, message):
+                self.event_summary = re.match(craft_pattern, message).groups()[0]
+            elif re.match(craft_pattern2, message):
+                self.event_summary = "a nearby crafting table"
+            elif re.match(mine_pattern, message):
+                self.event_summary = re.match(mine_pattern, message).groups()[0]
+            else:
+                self.event_summary = ""
+            return self.event_summary
+
+        chatlog = set()
+        for event_type, event in events:
+            if event_type == "onChat":
+                item = filter_item(event["onChat"])
+                if item:
+                    chatlog.add(item)
+        self.event_summary = "I also need " + ", ".join(chatlog) + "." if chatlog else ""
+
+    def reset_block_info(self):
+        # revert all the placing event in the last step
+        pass
+
+    def update_exploration_progress(self, success: bool):
+        """
+        Split task into completed_tasks or failed_tasks
+        Args: info = {
+            "task": self.task,
+            "success": success,
+            "conversations": self.conversations,
+        }
+        """
+        self.runtime_status = success
+        task = self.current_task
+        if task.startswith("Deposit useless items into the chest at"):
+            return
+        if success:
+            logger.info(f"Completed task {task}.")
+            self.completed_tasks.append(task)
+        else:
+            logger.info(f"Failed to complete task {task}. Skipping to next task.")
+            self.failed_tasks.append(task)
+            # when not success, below to update event!
+            # revert all the placing event in the last step
+            blocks = []
+            positions = []
+            for event_type, event in self.event:
+                if event_type == "onSave" and event["onSave"].endswith("_placed"):
+                    block = event["onSave"].split("_placed")[0]
+                    position = event["status"]["position"]
+                    blocks.append(block)
+                    positions.append(position)
+            new_events = self.step(
+                f"await givePlacedItemBack(bot, {json.dumps(blocks)}, {json.dumps(positions)})",
+                programs=self.programs,
+            )
+            self.event[-1][1]["inventory"] = new_events[-1][1]["inventory"]
+            self.event[-1][1]["voxels"] = new_events[-1][1]["voxels"]
+
+        self.save_sorted_tasks()
+
+    def save_sorted_tasks(self):
+        updated_completed_tasks = []
+        # record repeated failed tasks
+        updated_failed_tasks = self.failed_tasks
+        # dedup but keep order
+        for task in self.completed_tasks:
+            if task not in updated_completed_tasks:
+                updated_completed_tasks.append(task)
+
+        # remove completed tasks from failed tasks
+        for task in updated_completed_tasks:
+            while task in updated_failed_tasks:
+                updated_failed_tasks.remove(task)
+
+        self.completed_tasks = updated_completed_tasks
+        self.failed_tasks = updated_failed_tasks
+
+        # dump to json
+        write_json_file(f"{MC_CKPT_DIR}/curriculum/completed_tasks.json", self.completed_tasks)
+        write_json_file(f"{MC_CKPT_DIR}/curriculum/failed_tasks.json", self.failed_tasks)
+
+    async def on_event_retrieve(self, *args):
+        """
+        Retrieve Minecraft events.
+
+        Returns:
+            list: A list of Minecraft events.
+
+            Raises:
+                Exception: If there is an issue retrieving events.
+        """
+        try:
+            self.reset(
+                options={
+                    "mode": "soft",
+                    "wait_ticks": 20,
+                }
+            )
+            # difficulty = "easy" if len(self.completed_tasks) > 15 else "peaceful"
+            difficulty = "peaceful"
+
+            events = self.step("bot.chat(`/time set ${getNextTime()}`);\n" + f"bot.chat('/difficulty {difficulty}');")
+            self.update_event(events)
+            return events
+        except Exception as e:
+            time.sleep(3)  # wait for mineflayer to exit
+            # reset bot status here
+            events = self.reset(
+                options={
+                    "mode": "hard",
+                    "wait_ticks": 20,
+                    "inventory": self.event[-1][1]["inventory"],
+                    "equipment": self.event[-1][1]["status"]["equipment"],
+                    "position": self.event[-1][1]["status"]["position"],
+                }
+            )
+            self.update_event(events)
+            logger.error(f"Failed to retrieve Minecraft events: {str(e)}")
+            return events
+
+    async def on_event_execute(self, *args):
+        """
+        Execute Minecraft events.
+
+        This function is used to obtain events from the Minecraft environment. Check the implementation in
+        the 'voyager/env/bridge.py step()' function to capture events generated within the game.
+
+        Returns:
+            list: A list of Minecraft events.
+
+            Raises:
+                Exception: If there is an issue retrieving events.
+        """
+        try:
+            events = self.step(
+                code=self.code,
+                programs=self.programs,
+            )
+            self.update_event(events)
+            return events
+        except Exception as e:
+            time.sleep(3)  # wait for mineflayer to exit
+            # reset bot status here
+            events = self.reset(
+                options={
+                    "mode": "hard",
+                    "wait_ticks": 20,
+                    "inventory": self.event[-1][1]["inventory"],
+                    "equipment": self.event[-1][1]["status"]["equipment"],
+                    "position": self.event[-1][1]["status"]["position"],
+                }
+            )
+            self.update_event(events)
+            logger.error(f"Failed to execute Minecraft events: {str(e)}")
+            return events

metagpt/environment/mincraft_env/mincraft_ext_env.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Desc   : The Mincraft external environment to integrate with Mincraft game
+#           refs to `voyager bridge.py`
+
+import json
+import time
+from typing import Optional
+
+import requests
+from pydantic import ConfigDict, Field, model_validator
+
+from metagpt.environment.base_env import ExtEnv, mark_as_writeable
+from metagpt.environment.mincraft_env.const import (
+    MC_CKPT_DIR,
+    MC_CORE_INVENTORY_ITEMS,
+    MC_CURRICULUM_OB,
+    MC_DEFAULT_WARMUP,
+    METAGPT_ROOT,
+)
+from metagpt.environment.mincraft_env.process_monitor import SubprocessMonitor
+from metagpt.logs import logger
+
+
+class MincraftExtEnv(ExtEnv):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    mc_port: Optional[int] = Field(default=None)
+    server_host: str = Field(default="http://127.0.0.1")
+    server_port: str = Field(default=3000)
+    request_timeout: int = Field(default=600)
+
+    mineflayer: Optional[SubprocessMonitor] = Field(default=None, validate_default=True)
+
+    has_reset: bool = Field(default=False)
+    reset_options: Optional[dict] = Field(default=None)
+    connected: bool = Field(default=False)
+    server_paused: bool = Field(default=False)
+    warm_up: dict = Field(default=dict())
+
+    @property
+    def server(self) -> str:
+        return f"{self.server_host}:{self.server_port}"
+
+    @model_validator(mode="after")
+    def _post_init_ext_env(self):
+        if not self.mineflayer:
+            self.mineflayer = SubprocessMonitor(
+                commands=[
+                    "node",
+                    METAGPT_ROOT.joinpath("metagpt", "environment", "mincraft_env", "mineflayer", "index.js"),
+                    str(self.server_port),
+                ],
+                name="mineflayer",
+                ready_match=r"Server started on port (\d+)",
+            )
+        if not self.warm_up:
+            warm_up = MC_DEFAULT_WARMUP
+            if "optional_inventory_items" in warm_up:
+                assert MC_CORE_INVENTORY_ITEMS is not None
+                # self.core_inv_items_regex = re.compile(MC_CORE_INVENTORY_ITEMS)
+                self.warm_up["optional_inventory_items"] = warm_up["optional_inventory_items"]
+            else:
+                self.warm_up["optional_inventory_items"] = 0
+            for key in MC_CURRICULUM_OB:
+                self.warm_up[key] = warm_up.get(key, MC_DEFAULT_WARMUP[key])
+            self.warm_up["nearby_blocks"] = 0
+            self.warm_up["inventory"] = 0
+            self.warm_up["completed_tasks"] = 0
+            self.warm_up["failed_tasks"] = 0
+
+        # init ckpt sub-forders
+        MC_CKPT_DIR.joinpath("curriculum/vectordb").mkdir(parents=True, exist_ok=True)
+        MC_CKPT_DIR.joinpath("action").mkdir(exist_ok=True)
+        MC_CKPT_DIR.joinpath("skill/code").mkdir(parents=True, exist_ok=True)
+        MC_CKPT_DIR.joinpath("skill/description").mkdir(exist_ok=True)
+        MC_CKPT_DIR.joinpath("skill/vectordb").mkdir(exist_ok=True)
+
+    def set_mc_port(self, mc_port: int):
+        self.mc_port = mc_port
+
+    @mark_as_writeable
+    def close(self) -> bool:
+        self.unpause()
+        if self.connected:
+            res = requests.post(f"{self.server}/stop")
+            if res.status_code == 200:
+                self.connected = False
+        self.mineflayer.stop()
+        return not self.connected
+
+    @mark_as_writeable
+    def check_process(self) -> dict:
+        retry = 0
+        while not self.mineflayer.is_running:
+            logger.info("Mineflayer process has exited, restarting")
+            self.mineflayer.run()
+            if not self.mineflayer.is_running:
+                if retry > 3:
+                    logger.error("Mineflayer process failed to start")
+                    raise {}
+                else:
+                    retry += 1
+                    continue
+            logger.info(self.mineflayer.ready_line)
+            res = requests.post(
+                f"{self.server}/start",
+                json=self.reset_options,
+                timeout=self.request_timeout,
+            )
+            if res.status_code != 200:
+                self.mineflayer.stop()
+                logger.error(f"Minecraft server reply with code {res.status_code}")
+                raise {}
+            return res.json()
+
+    @mark_as_writeable
+    def reset(self, *, seed=None, options=None) -> dict:
+        if options is None:
+            options = {}
+        if options.get("inventory", {}) and options.get("mode", "hard") != "hard":
+            logger.error("inventory can only be set when options is hard")
+            raise {}
+
+        self.reset_options = {
+            "port": self.mc_port,
+            "reset": options.get("mode", "hard"),
+            "inventory": options.get("inventory", {}),
+            "equipment": options.get("equipment", []),
+            "spread": options.get("spread", False),
+            "waitTicks": options.get("wait_ticks", 5),
+            "position": options.get("position", None),
+        }
+
+        self.unpause()
+        self.mineflayer.stop()
+        time.sleep(1)  # wait for mineflayer to exit
+
+        returned_data = self.check_process()
+        self.has_reset = True
+        self.connected = True
+        # All the reset in step will be soft
+        self.reset_options["reset"] = "soft"
+        self.pause()
+        return json.loads(returned_data)
+
+    @mark_as_writeable
+    def step(self, code: str, programs: str = "") -> dict:
+        if not self.has_reset:
+            raise RuntimeError("Environment has not been reset yet")
+        self.check_process()
+        self.unpause()
+        data = {
+            "code": code,
+            "programs": programs,
+        }
+        res = requests.post(f"{self.server}/step", json=data, timeout=self.request_timeout)
+        if res.status_code != 200:
+            raise RuntimeError("Failed to step Minecraft server")
+        returned_data = res.json()
+        self.pause()
+        return json.loads(returned_data)
+
+    @mark_as_writeable
+    def pause(self) -> bool:
+        if self.mineflayer.is_running and not self.server_paused:
+            res = requests.post(f"{self.server}/pause")
+            if res.status_code == 200:
+                self.server_paused = True
+        return self.server_paused
+
+    @mark_as_writeable
+    def unpause(self) -> bool:
+        if self.mineflayer.is_running and self.server_paused:
+            res = requests.post(f"{self.server}/pause")
+            if res.status_code == 200:
+                self.server_paused = False
+            else:
+                logger.info(f"mineflayer pause result: {res.json()}")
+        return self.server_paused

metagpt/environment/mincraft_env/mineflayer/.gitignore

@@ -0,0 +1 @@
+!/lib

metagpt/environment/mincraft_env/mineflayer/.prettierignore

@@ -0,0 +1,3 @@
+# Ignore artifacts:
+build
+coverage

metagpt/environment/mincraft_env/mineflayer/.prettierrc.json

@@ -0,0 +1,3 @@
+{
+    "tabWidth": 4
+}

metagpt/environment/mincraft_env/mineflayer/index.js

@@ -0,0 +1,425 @@
+const fs = require("fs");
+const express = require("express");
+const bodyParser = require("body-parser");
+const mineflayer = require("mineflayer");
+
+const skills = require("./lib/skillLoader");
+const { initCounter, getNextTime } = require("./lib/utils");
+const obs = require("./lib/observation/base");
+const OnChat = require("./lib/observation/onChat");
+const OnError = require("./lib/observation/onError");
+const { Voxels, BlockRecords } = require("./lib/observation/voxels");
+const Status = require("./lib/observation/status");
+const Inventory = require("./lib/observation/inventory");
+const OnSave = require("./lib/observation/onSave");
+const Chests = require("./lib/observation/chests");
+const { plugin: tool } = require("mineflayer-tool");
+
+let bot = null;
+
+const app = express();
+
+app.use(bodyParser.json({ limit: "50mb" }));
+app.use(bodyParser.urlencoded({ limit: "50mb", extended: false }));
+
+app.post("/start", (req, res) => {
+    if (bot) onDisconnect("Restarting bot");
+    bot = null;
+    console.log(req.body);
+    bot = mineflayer.createBot({
+        host: "localhost", // minecraft server ip
+        port: req.body.port, // minecraft server port
+        username: "bot",
+        disableChatSigning: true,
+        checkTimeoutInterval: 60 * 60 * 1000,
+    });
+    bot.once("error", onConnectionFailed);
+
+    // Event subscriptions
+    bot.waitTicks = req.body.waitTicks;
+    bot.globalTickCounter = 0;
+    bot.stuckTickCounter = 0;
+    bot.stuckPosList = [];
+    bot.iron_pickaxe = false;
+
+    bot.on("kicked", onDisconnect);
+
+    // mounting will cause physicsTick to stop
+    bot.on("mount", () => {
+        bot.dismount();
+    });
+
+    bot.once("spawn", async () => {
+        bot.removeListener("error", onConnectionFailed);
+        let itemTicks = 1;
+        if (req.body.reset === "hard") {
+            bot.chat("/clear @s");
+            bot.chat("/kill @s");
+            const inventory = req.body.inventory ? req.body.inventory : {};
+            const equipment = req.body.equipment
+                ? req.body.equipment
+                : [null, null, null, null, null, null];
+            for (let key in inventory) {
+                bot.chat(`/give @s minecraft:${key} ${inventory[key]}`);
+                itemTicks += 1;
+            }
+            const equipmentNames = [
+                "armor.head",
+                "armor.chest",
+                "armor.legs",
+                "armor.feet",
+                "weapon.mainhand",
+                "weapon.offhand",
+            ];
+            for (let i = 0; i < 6; i++) {
+                if (i === 4) continue;
+                if (equipment[i]) {
+                    bot.chat(
+                        `/item replace entity @s ${equipmentNames[i]} with minecraft:${equipment[i]}`
+                    );
+                    itemTicks += 1;
+                }
+            }
+        }
+
+        if (req.body.position) {
+            bot.chat(
+                `/tp @s ${req.body.position.x} ${req.body.position.y} ${req.body.position.z}`
+            );
+        }
+
+        // if iron_pickaxe is in bot's inventory
+        if (
+            bot.inventory.items().find((item) => item.name === "iron_pickaxe")
+        ) {
+            bot.iron_pickaxe = true;
+        }
+
+        const { pathfinder } = require("mineflayer-pathfinder");
+        const tool = require("mineflayer-tool").plugin;
+        const collectBlock = require("mineflayer-collectblock").plugin;
+        const pvp = require("mineflayer-pvp").plugin;
+        const minecraftHawkEye = require("minecrafthawkeye");
+        bot.loadPlugin(pathfinder);
+        bot.loadPlugin(tool);
+        bot.loadPlugin(collectBlock);
+        bot.loadPlugin(pvp);
+        bot.loadPlugin(minecraftHawkEye);
+
+        // bot.collectBlock.movements.digCost = 0;
+        // bot.collectBlock.movements.placeCost = 0;
+
+        obs.inject(bot, [
+            OnChat,
+            OnError,
+            Voxels,
+            Status,
+            Inventory,
+            OnSave,
+            Chests,
+            BlockRecords,
+        ]);
+        skills.inject(bot);
+
+        if (req.body.spread) {
+            bot.chat(`/spreadplayers ~ ~ 0 300 under 80 false @s`);
+            await bot.waitForTicks(bot.waitTicks);
+        }
+
+        await bot.waitForTicks(bot.waitTicks * itemTicks);
+        res.json(bot.observe());
+
+        initCounter(bot);
+        bot.chat("/gamerule keepInventory true");
+        bot.chat("/gamerule doDaylightCycle false");
+    });
+
+    function onConnectionFailed(e) {
+        console.log(e);
+        bot = null;
+        res.status(400).json({ error: e });
+    }
+    function onDisconnect(message) {
+        if (bot.viewer) {
+            bot.viewer.close();
+        }
+        bot.end();
+        console.log(message);
+        bot = null;
+    }
+});
+
+app.post("/step", async (req, res) => {
+    // import useful package
+    let response_sent = false;
+    function otherError(err) {
+        console.log("Uncaught Error");
+        bot.emit("error", handleError(err));
+        bot.waitForTicks(bot.waitTicks).then(() => {
+            if (!response_sent) {
+                response_sent = true;
+                res.json(bot.observe());
+            }
+        });
+    }
+
+    process.on("uncaughtException", otherError);
+
+    const mcData = require("minecraft-data")(bot.version);
+    mcData.itemsByName["leather_cap"] = mcData.itemsByName["leather_helmet"];
+    mcData.itemsByName["leather_tunic"] =
+        mcData.itemsByName["leather_chestplate"];
+    mcData.itemsByName["leather_pants"] =
+        mcData.itemsByName["leather_leggings"];
+    mcData.itemsByName["leather_boots"] = mcData.itemsByName["leather_boots"];
+    mcData.itemsByName["lapis_lazuli_ore"] = mcData.itemsByName["lapis_ore"];
+    mcData.blocksByName["lapis_lazuli_ore"] = mcData.blocksByName["lapis_ore"];
+    const {
+        Movements,
+        goals: {
+            Goal,
+            GoalBlock,
+            GoalNear,
+            GoalXZ,
+            GoalNearXZ,
+            GoalY,
+            GoalGetToBlock,
+            GoalLookAtBlock,
+            GoalBreakBlock,
+            GoalCompositeAny,
+            GoalCompositeAll,
+            GoalInvert,
+            GoalFollow,
+            GoalPlaceBlock,
+        },
+        pathfinder,
+        Move,
+        ComputedPath,
+        PartiallyComputedPath,
+        XZCoordinates,
+        XYZCoordinates,
+        SafeBlock,
+        GoalPlaceBlockOptions,
+    } = require("mineflayer-pathfinder");
+    const { Vec3 } = require("vec3");
+
+    // Set up pathfinder
+    const movements = new Movements(bot, mcData);
+    bot.pathfinder.setMovements(movements);
+
+    bot.globalTickCounter = 0;
+    bot.stuckTickCounter = 0;
+    bot.stuckPosList = [];
+
+    function onTick() {
+        bot.globalTickCounter++;
+        if (bot.pathfinder.isMoving()) {
+            bot.stuckTickCounter++;
+            if (bot.stuckTickCounter >= 100) {
+                onStuck(1.5);
+                bot.stuckTickCounter = 0;
+            }
+        }
+    }
+
+    bot.on("physicTick", onTick);
+
+    // initialize fail count
+    let _craftItemFailCount = 0;
+    let _killMobFailCount = 0;
+    let _mineBlockFailCount = 0;
+    let _placeItemFailCount = 0;
+    let _smeltItemFailCount = 0;
+
+    // Retrieve array form post bod
+    const code = req.body.code;
+    const programs = req.body.programs;
+    bot.cumulativeObs = [];
+    await bot.waitForTicks(bot.waitTicks);
+    const r = await evaluateCode(code, programs);
+    process.off("uncaughtException", otherError);
+    if (r !== "success") {
+        bot.emit("error", handleError(r));
+    }
+    await returnItems();
+    // wait for last message
+    await bot.waitForTicks(bot.waitTicks);
+    if (!response_sent) {
+        response_sent = true;
+        res.json(bot.observe());
+    }
+    bot.removeListener("physicTick", onTick);
+
+    async function evaluateCode(code, programs) {
+        // Echo the code produced for players to see it. Don't echo when the bot code is already producing dialog or it will double echo
+        try {
+            await eval("(async () => {" + programs + "\n" + code + "})()");
+            return "success";
+        } catch (err) {
+            return err;
+        }
+    }
+
+    function onStuck(posThreshold) {
+        const currentPos = bot.entity.position;
+        bot.stuckPosList.push(currentPos);
+
+        // Check if the list is full
+        if (bot.stuckPosList.length === 5) {
+            const oldestPos = bot.stuckPosList[0];
+            const posDifference = currentPos.distanceTo(oldestPos);
+
+            if (posDifference < posThreshold) {
+                teleportBot(); // execute the function
+            }
+
+            // Remove the oldest time from the list
+            bot.stuckPosList.shift();
+        }
+    }
+
+    function teleportBot() {
+        const blocks = bot.findBlocks({
+            matching: (block) => {
+                return block.type === 0;
+            },
+            maxDistance: 1,
+            count: 27,
+        });
+
+        if (blocks) {
+            // console.log(blocks.length);
+            const randomIndex = Math.floor(Math.random() * blocks.length);
+            const block = blocks[randomIndex];
+            bot.chat(`/tp @s ${block.x} ${block.y} ${block.z}`);
+        } else {
+            bot.chat("/tp @s ~ ~1.25 ~");
+        }
+    }
+
+    function returnItems() {
+        bot.chat("/gamerule doTileDrops false");
+        const crafting_table = bot.findBlock({
+            matching: mcData.blocksByName.crafting_table.id,
+            maxDistance: 128,
+        });
+        if (crafting_table) {
+            bot.chat(
+                `/setblock ${crafting_table.position.x} ${crafting_table.position.y} ${crafting_table.position.z} air destroy`
+            );
+            bot.chat("/give @s crafting_table");
+        }
+        const furnace = bot.findBlock({
+            matching: mcData.blocksByName.furnace.id,
+            maxDistance: 128,
+        });
+        if (furnace) {
+            bot.chat(
+                `/setblock ${furnace.position.x} ${furnace.position.y} ${furnace.position.z} air destroy`
+            );
+            bot.chat("/give @s furnace");
+        }
+        if (bot.inventoryUsed() >= 32) {
+            // if chest is not in bot's inventory
+            if (!bot.inventory.items().find((item) => item.name === "chest")) {
+                bot.chat("/give @s chest");
+            }
+        }
+        // if iron_pickaxe not in bot's inventory and bot.iron_pickaxe
+        if (
+            bot.iron_pickaxe &&
+            !bot.inventory.items().find((item) => item.name === "iron_pickaxe")
+        ) {
+            bot.chat("/give @s iron_pickaxe");
+        }
+        bot.chat("/gamerule doTileDrops true");
+    }
+
+    function handleError(err) {
+        let stack = err.stack;
+        if (!stack) {
+            return err;
+        }
+        console.log(stack);
+        const final_line = stack.split("\n")[1];
+        const regex = /<anonymous>:(\d+):\d+\)/;
+
+        const programs_length = programs.split("\n").length;
+        let match_line = null;
+        for (const line of stack.split("\n")) {
+            const match = regex.exec(line);
+            if (match) {
+                const line_num = parseInt(match[1]);
+                if (line_num >= programs_length) {
+                    match_line = line_num - programs_length;
+                    break;
+                }
+            }
+        }
+        if (!match_line) {
+            return err.message;
+        }
+        let f_line = final_line.match(
+            /\((?<file>.*):(?<line>\d+):(?<pos>\d+)\)/
+        );
+        if (f_line && f_line.groups && fs.existsSync(f_line.groups.file)) {
+            const { file, line, pos } = f_line.groups;
+            const f = fs.readFileSync(file, "utf8").split("\n");
+            // let filename = file.match(/(?<=node_modules\\)(.*)/)[1];
+            let source = file + `:${line}\n${f[line - 1].trim()}\n `;
+
+            const code_source =
+                "at " +
+                code.split("\n")[match_line - 1].trim() +
+                " in your code";
+            return source + err.message + "\n" + code_source;
+        } else if (
+            f_line &&
+            f_line.groups &&
+            f_line.groups.file.includes("<anonymous>")
+        ) {
+            const { file, line, pos } = f_line.groups;
+            let source =
+                "Your code" +
+                `:${match_line}\n${code.split("\n")[match_line - 1].trim()}\n `;
+            let code_source = "";
+            if (line < programs_length) {
+                source =
+                    "In your program code: " +
+                    programs.split("\n")[line - 1].trim() +
+                    "\n";
+                code_source = `at line ${match_line}:${code
+                    .split("\n")
+                    [match_line - 1].trim()} in your code`;
+            }
+            return source + err.message + "\n" + code_source;
+        }
+        return err.message;
+    }
+});
+
+app.post("/stop", (req, res) => {
+    bot.end();
+    res.json({
+        message: "Bot stopped",
+    });
+});
+
+app.post("/pause", (req, res) => {
+    if (!bot) {
+        res.status(400).json({ error: "Bot not spawned" });
+        return;
+    }
+    bot.chat("/pause");
+    bot.waitForTicks(bot.waitTicks).then(() => {
+        res.json({ message: "Success" });
+    });
+});
+
+// Server listening to PORT 3000
+
+const DEFAULT_PORT = 3000;
+const PORT = process.argv[2] || DEFAULT_PORT;
+app.listen(PORT, () => {
+    console.log(`Server started on port ${PORT}`);
+});

metagpt/environment/mincraft_env/mineflayer/lib/observation/base.js

@@ -0,0 +1,45 @@
+class Observation {
+    constructor(bot) {
+        if (new.target === Observation) {
+            throw new TypeError(
+                "Cannot instantiate abstract class Observation"
+            );
+        }
+
+        this.bot = bot;
+        this.name = "Observation";
+    }
+
+    observe() {
+        throw new TypeError("Method 'observe()' must be implemented.");
+    }
+
+    reset() {}
+}
+
+function inject(bot, obs_list) {
+    bot.obsList = [];
+    bot.cumulativeObs = [];
+    bot.eventMemory = {};
+    obs_list.forEach((obs) => {
+        bot.obsList.push(new obs(bot));
+    });
+    bot.event = function (event_name) {
+        let result = {};
+        bot.obsList.forEach((obs) => {
+            if (obs.name.startsWith("on") && obs.name !== event_name) {
+                return;
+            }
+            result[obs.name] = obs.observe();
+        });
+        bot.cumulativeObs.push([event_name, result]);
+    };
+    bot.observe = function () {
+        bot.event("observe");
+        const result = bot.cumulativeObs;
+        bot.cumulativeObs = [];
+        return JSON.stringify(result);
+    };
+}
+
+module.exports = { Observation, inject };

metagpt/environment/mincraft_env/mineflayer/lib/observation/chests.js

@@ -0,0 +1,31 @@
+const { Observation } = require("./base");
+
+class Chests extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "nearbyChests";
+        this.chestsItems = {};
+        bot.on("closeChest", (chestItems, position) => {
+            this.chestsItems[position] = chestItems;
+        });
+        bot.on("removeChest", (chestPosition) => {
+            this.chestsItems[chestPosition] = "Invalid";
+        });
+    }
+
+    observe() {
+        const chests = this.bot.findBlocks({
+            matching: this.bot.registry.blocksByName.chest.id,
+            maxDistance: 16,
+            count: 999,
+        });
+        chests.forEach((chest) => {
+            if (!this.chestsItems.hasOwnProperty(chest)) {
+                this.chestsItems[chest] = "Unknown";
+            }
+        });
+        return this.chestsItems;
+    }
+}
+
+module.exports = Chests;

metagpt/environment/mincraft_env/mineflayer/lib/observation/inventory.js

@@ -0,0 +1,39 @@
+const { Observation } = require("./base");
+
+class Inventory extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "inventory";
+    }
+
+    observe() {
+        return listItems(this.bot);
+    }
+}
+
+function listItems(bot) {
+    const items = getInventoryItems(bot);
+    return items.reduce(itemToDict, {});
+}
+
+function getInventoryItems(bot) {
+    const inventory = bot.currentWindow || bot.inventory;
+    return inventory.items();
+}
+
+function itemToDict(acc, cur) {
+    if (cur.name && cur.count) {
+        //if both name and count property are defined
+        if (acc[cur.name]) {
+            //if the item is already in the dict
+            acc[cur.name] += cur.count;
+        } else {
+            //if the item is not in the dict
+            acc[cur.name] = cur.count;
+        }
+    }
+    return acc;
+}
+
+//export modules
+module.exports = Inventory;

metagpt/environment/mincraft_env/mineflayer/lib/observation/onChat.js

@@ -0,0 +1,26 @@
+const Observation = require("./base.js").Observation;
+
+class onChat extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "onChat";
+        this.obs = "";
+        bot.on("chatEvent", (username, message) => {
+            // Save entity status to local variable
+            if (message.startsWith("/")) {
+                return;
+            }
+
+            this.obs += message;
+            this.bot.event(this.name);
+        });
+    }
+
+    observe() {
+        const result = this.obs;
+        this.obs = "";
+        return result;
+    }
+}
+
+module.exports = onChat;

metagpt/environment/mincraft_env/mineflayer/lib/observation/onError.js

@@ -0,0 +1,22 @@
+const Observation = require("./base.js").Observation;
+
+class onError extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "onError";
+        this.obs = null;
+        bot.on("error", (err) => {
+            // Save entity status to local variable
+            this.obs = err;
+            this.bot.event(this.name);
+        });
+    }
+
+    observe() {
+        const result = this.obs;
+        this.obs = null;
+        return result;
+    }
+}
+
+module.exports = onError;

metagpt/environment/mincraft_env/mineflayer/lib/observation/onSave.js

@@ -0,0 +1,22 @@
+const Observation = require("./base.js").Observation;
+
+class onSave extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "onSave";
+        this.obs = null;
+        bot.on("save", (eventName) => {
+            // Save entity status to local variable
+            this.obs = eventName;
+            this.bot.event(this.name);
+        });
+    }
+
+    observe() {
+        const result = this.obs;
+        this.obs = null;
+        return result;
+    }
+}
+
+module.exports = onSave;

metagpt/environment/mincraft_env/mineflayer/lib/observation/status.js

@@ -0,0 +1,103 @@
+const Observation = require("./base.js").Observation;
+
+class Status extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "status";
+    }
+
+    observe() {
+        return {
+            health: this.bot.health,
+            food: this.bot.food,
+            saturation: this.bot.foodSaturation,
+            oxygen: this.bot.oxygenLevel,
+            position: this.bot.entity.position,
+            velocity: this.bot.entity.velocity,
+            yaw: this.bot.entity.yaw,
+            pitch: this.bot.entity.pitch,
+            onGround: this.bot.entity.onGround,
+            equipment: this.getEquipment(),
+            name: this.bot.entity.username,
+            timeSinceOnGround: this.bot.entity.timeSinceOnGround,
+            isInWater: this.bot.entity.isInWater,
+            isInLava: this.bot.entity.isInLava,
+            isInWeb: this.bot.entity.isInWeb,
+            isCollidedHorizontally: this.bot.entity.isCollidedHorizontally,
+            isCollidedVertically: this.bot.entity.isCollidedVertically,
+            biome: this.bot.blockAt(this.bot.entity.position)
+                ? this.bot.blockAt(this.bot.entity.position).biome.name
+                : "None",
+            entities: this.getEntities(),
+            timeOfDay: this.getTime(),
+            inventoryUsed: this.bot.inventoryUsed(),
+            elapsedTime: this.bot.globalTickCounter,
+        };
+    }
+
+    itemToObs(item) {
+        if (!item) return null;
+        return item.name;
+    }
+
+    getTime() {
+        const timeOfDay = this.bot.time.timeOfDay;
+        let time = "";
+        if (timeOfDay < 1000) {
+            time = "sunrise";
+        } else if (timeOfDay < 6000) {
+            time = "day";
+        } else if (timeOfDay < 12000) {
+            time = "noon";
+        } else if (timeOfDay < 13000) {
+            time = "sunset";
+        } else if (timeOfDay < 18000) {
+            time = "night";
+        } else if (timeOfDay < 22000) {
+            time = "midnight";
+        } else {
+            time = "sunrise";
+        }
+        return time;
+    }
+
+    // For each item in equipment, if it exists, return the name of the item
+    // otherwise return null
+    getEquipment() {
+        const slots = this.bot.inventory.slots;
+        const mainHand = this.bot.heldItem;
+        return slots
+            .slice(5, 9)
+            .concat(mainHand, slots[45])
+            .map(this.itemToObs);
+    }
+
+    getEntities() {
+        const entities = this.bot.entities;
+        if (!entities) return {};
+        // keep all monsters in one list, keep other mobs in another list
+        const mobs = {};
+        for (const id in entities) {
+            const entity = entities[id];
+            if (!entity.displayName) continue;
+            if (entity.name === "player" || entity.name === "item") continue;
+            if (entity.position.distanceTo(this.bot.entity.position) < 32) {
+                if (!mobs[entity.name]) {
+                    mobs[entity.name] = entity.position.distanceTo(
+                        this.bot.entity.position
+                    );
+                } else if (
+                    mobs[entity.name] >
+                    entity.position.distanceTo(this.bot.entity.position)
+                ) {
+                    mobs[entity.name] = entity.position.distanceTo(
+                        this.bot.entity.position
+                    );
+                }
+            }
+        }
+        return mobs;
+    }
+}
+
+module.exports = Status;

metagpt/environment/mincraft_env/mineflayer/lib/observation/voxels.js

@@ -0,0 +1,67 @@
+// Blocks = require("./blocks")
+const { Observation } = require("./base");
+
+class Voxels extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "voxels";
+    }
+
+    observe() {
+        return Array.from(getSurroundingBlocks(this.bot, 8, 2, 8));
+    }
+}
+
+class BlockRecords extends Observation {
+    constructor(bot) {
+        super(bot);
+        this.name = "blockRecords";
+        this.records = new Set();
+        this.tick = 0;
+        bot.on("physicsTick", () => {
+            this.tick++;
+            if (this.tick >= 100) {
+                const items = getInventoryItems(this.bot);
+                getSurroundingBlocks(this.bot, 8, 2, 8).forEach((block) => {
+                    if (!items.has(block)) this.records.add(block);
+                });
+                this.tick = 0;
+            }
+        });
+    }
+
+    observe() {
+        return Array.from(this.records);
+    }
+
+    reset() {
+        this.records = new Set();
+    }
+}
+
+function getSurroundingBlocks(bot, x_distance, y_distance, z_distance) {
+    const surroundingBlocks = new Set();
+
+    for (let x = -x_distance; x <= x_distance; x++) {
+        for (let y = -y_distance; y <= y_distance; y++) {
+            for (let z = -z_distance; z <= z_distance; z++) {
+                const block = bot.blockAt(bot.entity.position.offset(x, y, z));
+                if (block && block.type !== 0) {
+                    surroundingBlocks.add(block.name);
+                }
+            }
+        }
+    }
+    // console.log(surroundingBlocks);
+    return surroundingBlocks;
+}
+
+function getInventoryItems(bot) {
+    const items = new Set();
+    bot.inventory.items().forEach((item) => {
+        if (item) items.add(item.name);
+    });
+    return items;
+}
+
+module.exports = { Voxels, BlockRecords };

metagpt/environment/mincraft_env/mineflayer/lib/skillLoader.js

@@ -0,0 +1,79 @@
+function inject(bot) {
+    bot._sleep = bot.sleep;
+    bot.sleep = async (bedBlock) => {
+        await bot.waitForTicks(20);
+        await bot._sleep(bedBlock);
+        await bot.waitForTicks(135);
+    };
+
+    bot._fish = bot.fish;
+    bot.fish = async () => {
+        if (bot.heldItem?.name !== "fishing_rod") {
+            bot.chat("I'm not holding a fishing rod!");
+            return;
+        }
+        let timeout = null;
+        await Promise.race([
+            bot._fish(),
+            new Promise(
+                (resolve, reject) =>
+                    (timeout = setTimeout(() => {
+                        bot.activateItem();
+                        reject(
+                            new Error(
+                                "Finishing timeout, make sure you get to and look at a water block!"
+                            )
+                        );
+                    }, 60000))
+            ),
+        ]);
+        clearTimeout(timeout);
+        await bot.waitForTicks(20);
+    };
+
+    bot._consume = bot.consume;
+    bot.consume = async () => {
+        // action_count.activateItem++;
+        await bot._consume();
+        await bot.waitForTicks(20);
+    };
+
+    bot._useOn = bot.useOn;
+    bot.useOn = async (entity) => {
+        if (entity.position.distanceTo(bot.entity.position) > 6) {
+            bot.chat("Please goto a place near the entity first!");
+            return;
+        }
+        await bot._useOn(entity);
+        await bot.waitForTicks(20);
+    };
+
+    bot._activateBlock = bot.activateBlock;
+    bot.activateBlock = async (block) => {
+        if (block.position.distanceTo(bot.entity.position) > 6) {
+            bot.chat("Please goto a place near the block first!");
+            return;
+        }
+        // action_count.activateBlock++;
+        await bot._activateBlock(block);
+    };
+
+    bot._chat = bot.chat;
+    bot.chat = (message) => {
+        // action_count.chat++;
+        bot.emit("chatEvent", "bot", message);
+        bot._chat(message);
+    };
+
+    bot.inventoryUsed = () => {
+        return bot.inventory.slots.slice(9, 45).filter((item) => item !== null)
+            .length;
+    };
+
+    bot.save = function (eventName) {
+        bot.emit("save", eventName);
+    };
+}
+
+// export all control_primitives
+module.exports = { inject };

metagpt/environment/mincraft_env/mineflayer/lib/utils.js

@@ -0,0 +1,31 @@
+let gameTimeCounter = 0;
+let gameTimeList = [];
+const initCounter = (bot) => {
+    gameTimeList = [];
+    for (let i = 0; i < 13000; i += 1000) {
+        gameTimeList.push(i);
+    }
+    for (let i = 13000; i < 24000; i += 2000) {
+        gameTimeList.push(i);
+    }
+    const timeOfDay = bot.time.timeOfDay;
+    for (let i = 0; i < gameTimeList.length; i++) {
+        if (gameTimeList[i] > timeOfDay) {
+            gameTimeCounter = i - 1;
+            break;
+        }
+    }
+};
+
+const getNextTime = () => {
+    gameTimeCounter++;
+    if (gameTimeCounter >= gameTimeList.length) {
+        gameTimeCounter = 0;
+    }
+    return gameTimeList[gameTimeCounter];
+};
+
+module.exports = {
+    initCounter,
+    getNextTime,
+};

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/.gitignore

@@ -0,0 +1,107 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Diagnostic reports (https://nodejs.org/api/report.html)
+report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (https://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules/
+jspm_packages/
+
+# TypeScript v1 declaration files
+typings/
+
+# TypeScript cache
+*.tsbuildinfo
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+
+# Next.js build output
+.next
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+# Comment in the public line in if your project uses Gatsby and *not* Next.js
+# https://nextjs.org/blog/next-9-1#public-directory-support
+# public
+
+# vuepress build output
+.vuepress/dist
+
+# Serverless directories
+.serverless/
+
+# FuseBox cache
+.fusebox/
+
+# DynamoDB Local files
+.dynamodb/
+
+# TernJS port file
+.tern-port
+
+lib/
+package-lock.json

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 TheDudeFromCI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/README.md

@@ -0,0 +1,89 @@
+<h1 align="center">mineflayer-collectblock</h1>
+<p align="center"><i>A small utility plugin for allowing users to collect blocks using a higher level API.</i></p>
+
+<p align="center">
+  <img src="https://github.com/TheDudeFromCI/mineflayer-collectblock/workflows/Build/badge.svg" />
+  <a href="https://www.npmjs.com/package/mineflayer-collectblock"><img src="https://img.shields.io/npm/v/mineflayer-collectblock" /></a>
+  <img src="https://img.shields.io/github/repo-size/TheDudeFromCI/mineflayer-collectblock" />
+  <img src="https://img.shields.io/npm/dm/mineflayer-collectblock" />
+  <img src="https://img.shields.io/github/contributors/TheDudeFromCI/mineflayer-collectblock" />
+  <img src="https://img.shields.io/github/license/TheDudeFromCI/mineflayer-collectblock" />
+</p>
+
+---
+## This is a modified version to better support Voyager
+
+## Showcase
+
+You can see a video of the plugin in action, [here.](https://youtu.be/5T_rcCnNnf4)
+The source code of the bot in the video can be seen in the examples folder, [here.](https://github.com/TheDudeFromCI/mineflayer-collectblock/blob/master/examples/collector.js)
+
+### Description
+
+This plugin is a wrapper for mineflayer that allows for easier API usage when collecting blocks or item drops. This plugin is designed to reduce some of the boilerplate code based around the act of pathfinding to a block _(handled by_ ***mineflayer-pathfinder***_)_, selecting the best tool to mine that block _(handled by_ ***mineflayer-tool***_)_, actually mining it, then moving to collect the item drops from that block. This plugin allows for all of that basic concept to be wrapped up into a single API function.
+
+In addition to the usage above, some additional quality of life features are available in this plugin. These include the ability to automatically deposit items into a chest when the bot's inventory is full, collecting new tools from a chest if the bot doesn't currently have a required tool _(also handled by_ ***mineflayer-tool***_)_, and allowing for queueing of multiple blocks or item drops to the collection task, so they can be processed later.
+
+### Getting Started
+
+This plugin is built using Node and can be installed using:
+```bash
+npm install --save mineflayer-collectblock
+```
+
+### Simple Bot
+
+The brief description goes here.
+
+```js
+// Create your bot
+const mineflayer = require("mineflayer")
+const bot = mineflayer.createBot({
+  host: 'localhost',
+  username: 'Player',
+})
+let mcData
+
+// Load collect block
+bot.loadPlugin(require('mineflayer-collectblock').plugin)
+
+async function collectGrass() {
+  // Find a nearby grass block
+  const grass = bot.findBlock({
+    matching: mcData.blocksByName.grass_block.id,
+    maxDistance: 64
+  })
+
+  if (grass) {
+    // If we found one, collect it.
+    try {
+      await bot.collectBlock.collect(grass)
+      collectGrass() // Collect another grass block
+    } catch (err) {
+      console.log(err) // Handle errors, if any
+    }
+  }
+}
+
+// On spawn, start collecting all nearby grass
+bot.once('spawn', () => {
+  mcData = require('minecraft-data')(bot.version)
+  collectGrass()
+})
+```
+
+### Documentation
+
+[API](https://github.com/TheDudeFromCI/mineflayer-collectblock/blob/master/docs/api.md)
+
+[Examples](https://github.com/TheDudeFromCI/mineflayer-collectblock/tree/master/examples)
+
+### License
+
+This project uses the [MIT](https://github.com/TheDudeFromCI/mineflayer-collectblock/blob/master/LICENSE) license.
+
+### Contributions
+
+This project is accepting PRs and Issues. See something you think can be improved? Go for it! Any and all help is highly appreciated!
+
+For larger changes, it is recommended to discuss these changes in the issues tab before writing any code. It's also preferred to make many smaller PRs than one large one, where applicable.

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-cayman

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/docs/api.md

@@ -0,0 +1,52 @@
+# API <!-- omit in toc -->
+
+Welcome to the *mineflayer-collectblock* API documentation page.
+
+## Table of Contents <!-- omit in toc -->
+
+- [1. Summary](#1-summary)
+- [Properties](#properties)
+  - [`bot.collectblock.movements: Movements`](#botcollectblockmovements-movements)
+- [Functions](#functions)
+  - [collect](#collect)
+    - [Options:](#options)
+
+## 1. Summary
+
+The collect block plugin is a utility plugin that can be used to help make collecting blocks and item drops very easy, using only a single API call. No need to worry about pathfinding to the block, selecting the right tool, or moving to pick up the item drop after mining.
+
+## Properties
+
+### `bot.collectblock.movements: Movements`
+
+The movements object used by the pathfinder plugin to define the movement configuration. This object is passed to the pathfinder plugin when any API from this plugin is called in order to control how pathfinding should work when collecting the given blocks or item.
+
+If set to null, the pathfinder plugin movements is not updated.
+
+Defaults to a new movements object instance.
+
+## Functions
+
+### collect
+
+Usage: `bot.collectblock.collect(target: Collectable | Collectable[], options?: CollectOptions, cb: (err?: Error) => void): void`
+
+Causes the bot to collect the given block, item drop, or list of those. If the target is a block, the bot will move to the block, mine it, and pick up the item drop. If the target is an item drop, the bot will move to the item drop and pick it up. If the target is a list of collectables, the bot will move from target to target in order of closest to furthest and collect each target in turn.
+
+#### Options:
+
+  * `append: boolean`
+
+    If true, the target(s) will be appended to the existing target list instead of starting a new task. Defaults to false.
+
+  * `ignoreNoPath: boolean`
+
+    If true, errors will not be thrown when a path to the target block cannot be found. The bot will attempt to choose the best available position it can find, instead. Errors are still thrown if the bot cannot interact with the block from it's final location. Defaults to false.
+
+  * `chestLocations: Vec3[]`
+
+    Gets the list of chest locations to use when storing items after the bot's inventory becomes full. If undefined, it defaults to the chest location list on the bot.collectBlock plugin.
+
+  * `itemFilter: ItemFilter`
+
+    When transferring items to a chest, this filter is used to determine what items are allowed to be moved, and what items aren't allowed to be moved. Defaults to the item filter specified on the bot.collectBlock plugin.

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/examples/collector.js

@@ -0,0 +1,70 @@
+/**
+ * This bot example show how to direct a bot to collect a specific block type
+ * or a group of nearby blocks of that type.
+ */
+
+const mineflayer = require('mineflayer')
+const collectBlock = require('mineflayer-collectblock').plugin
+
+if (process.argv.length < 4 || process.argv.length > 6) {
+  console.log('Usage : node collector.js <host> <port> [<name>] [<password>]')
+  process.exit(1)
+}
+
+const bot = mineflayer.createBot({
+  host: process.argv[2],
+  port: process.argv[3],
+  username: process.argv[4] || 'collector',
+  password: process.argv[5]
+})
+
+bot.loadPlugin(collectBlock)
+
+let mcData
+bot.once('spawn', () => {
+  mcData = require('minecraft-data')(bot.version)
+})
+
+bot.on('chat', async (username, message) => {
+  const args = message.split(' ')
+  if (args[0] !== 'collect') return
+
+  let count = 1
+  if (args.length === 3) count = parseInt(args[1])
+
+  let type = args[1]
+  if (args.length === 3) type = args[2]
+
+  const blockType = mcData.blocksByName[type]
+  if (!blockType) {
+    return
+  }
+
+  const blocks = bot.findBlocks({
+    matching: blockType.id,
+    maxDistance: 64,
+    count: count
+  })
+
+  if (blocks.length === 0) {
+    bot.chat("I don't see that block nearby.")
+    return
+  }
+
+  const targets = []
+  for (let i = 0; i < Math.min(blocks.length, count); i++) {
+    targets.push(bot.blockAt(blocks[i]))
+  }
+
+  bot.chat(`Found ${targets.length} ${type}(s)`)
+
+  try {
+    await bot.collectBlock.collect(targets)
+    // All blocks have been collected.
+    bot.chat('Done')
+  } catch (err) {
+    // An error occurred, report it.
+    bot.chat(err.message)
+    console.log(err)
+  }
+})

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/examples/oreMiner.js

@@ -0,0 +1,59 @@
+/**
+ * This bot example shows how to collect a vein of ores quickly after only finding a single block.
+ * This makes it easy to collect a vein of ores or mine a tree without looking for every block in the
+ * area.
+ */
+
+const mineflayer = require('mineflayer')
+const collectBlock = require('mineflayer-collectblock').plugin
+
+if (process.argv.length < 4 || process.argv.length > 6) {
+  console.log('Usage : node oreMiner.js <host> <port> [<name>] [<password>]')
+  process.exit(1)
+}
+
+const bot = mineflayer.createBot({
+  host: process.argv[2],
+  port: process.argv[3],
+  username: process.argv[4] || 'oreMiner',
+  password: process.argv[5]
+})
+
+bot.loadPlugin(collectBlock)
+
+let mcData
+bot.once('spawn', () => {
+  mcData = require('minecraft-data')(bot.version)
+})
+
+bot.on('chat', async (username, message) => {
+  const args = message.split(' ')
+  if (args[0] !== 'collect') return
+
+  const blockType = mcData.blocksByName[args[1]]
+  if (!blockType) {
+    bot.chat(`I don't know any blocks named ${args[1]}.`)
+    return
+  }
+
+  const block = bot.findBlock({
+    matching: blockType.id,
+    maxDistance: 64
+  })
+
+  if (!block) {
+    bot.chat("I don't see that block nearby.")
+    return
+  }
+
+  const targets = bot.collectBlock.findFromVein(block)
+  try {
+    await bot.collectBlock.collect(targets)
+    // All blocks have been collected.
+    bot.chat('Done')
+  } catch (err) {
+    // An error occurred, report it.
+    bot.chat(err.message)
+    console.log(err)
+  }
+})

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/examples/storageBot.js

@@ -0,0 +1,107 @@
+/**
+ * This bot example shows how to use the chest filling mechanic of the plugin.
+ * Simply provide a given storage chest, and the bot will automatically try and
+ * store it's inventory in that chest when the bot's inventory becomes full.
+ */
+
+if (process.argv.length < 4 || process.argv.length > 6) {
+  console.log('Usage : node storageBot.js <host> <port> [<name>] [<password>]')
+  process.exit(1)
+}
+
+// Load your libraries
+const mineflayer = require('mineflayer')
+const collectBlock = require('mineflayer-collectblock').plugin
+
+// Create your bot
+const bot = mineflayer.createBot({
+  host: process.argv[2],
+  port: parseInt(process.argv[3]),
+  username: process.argv[4] ? process.argv[4] : 'storageBot',
+  password: process.argv[5]
+})
+
+// Load the collect block plugin
+bot.loadPlugin(collectBlock)
+
+// Load mcData on login
+let mcData
+bot.once('login', () => {
+  mcData = require('minecraft-data')(bot.version)
+})
+
+// On spawn, try to find any nearby chests and save those as storage locations.
+// When the bot's inventory becomes too full, it will empty it's inventory into
+// these chests before collecting more resources. If a chest gets full, it moves
+// to the next one in order until it's inventory is empty or it runs out of chests.
+bot.once('spawn', () => {
+  bot.collectBlock.chestLocations = bot.findBlocks({
+    matching: mcData.blocksByName.chest.id,
+    maxDistance: 16,
+    count: 999999 // Get as many chests as we can
+  })
+
+  if (bot.collectBlock.chestLocations.length === 0) {
+    bot.chat("I don't see any chests nearby.")
+  } else {
+    for (const chestPos of bot.collectBlock.chestLocations) {
+      bot.chat(`I found a chest at ${chestPos}`)
+    }
+  }
+})
+
+// Wait for someone to say something
+bot.on('chat', async (username, message) => {
+  // If the player says something start starts with "collect"
+  // Otherwise, do nothing
+  const args = message.split(' ')
+  if (args[0] !== 'collect') return
+
+  // If the player specifies a number, collect that many. Otherwise, default to 1.
+  let count = 1
+  if (args.length === 3) count = parseInt(args[1])
+
+  // If a number was given the item number is the 3rd arg, not the 2nd.
+  let type = args[1]
+  if (args.length === 3) type = args[2]
+
+  // Get the id of that block type for this version of Minecraft.
+  const blockType = mcData.blocksByName[type]
+  if (!blockType) {
+    bot.chat(`I don't know any blocks named ${type}.`)
+    return
+  }
+
+  // Find all nearby blocks of that type, up to the given count, within 64 blocks.
+  const blocks = bot.findBlocks({
+    matching: blockType.id,
+    maxDistance: 64,
+    count: count
+  })
+
+  // Complain if we can't find any nearby blocks of that type.
+  if (blocks.length === 0) {
+    bot.chat("I don't see that block nearby.")
+    return
+  }
+
+  // Convert the block position array into a block array to pass to collect block.
+  const targets = []
+  for (let i = 0; i < Math.min(blocks.length, count); i++) {
+    targets.push(bot.blockAt(blocks[i]))
+  }
+
+  // Announce what we found.
+  bot.chat(`Found ${targets.length} ${type}(s)`)
+
+  // Tell the bot to collect all of the given blocks in the block list.
+  try {
+    await bot.collectBlock.collect(targets)
+    // All blocks have been collected.
+    bot.chat('Done')
+  } catch (err) {
+    // An error occurred, report it.
+    bot.chat(err.message)
+    console.log(err)
+  }
+})

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "mineflayer-collectblock",
+  "version": "1.4.1",
+  "description": "A simple utility plugin for Mineflayer that add a higher level API for collecting blocks.",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "scripts": {
+    "build": "ts-standard && tsc && require-self",
+    "clean": "rm -rf lib",
+    "test": "test"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/TheDudeFromCI/mineflayer-collectblock.git"
+  },
+  "keywords": [
+    "mineflayer",
+    "plugin",
+    "api",
+    "utility",
+    "helper",
+    "collect"
+  ],
+  "author": "TheDudeFromCI",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/TheDudeFromCI/mineflayer-collectblock/issues"
+  },
+  "homepage": "https://github.com/TheDudeFromCI/mineflayer-collectblock#readme",
+  "dependencies": {
+    "mineflayer": "^4.0.0",
+    "mineflayer-pathfinder": "^2.1.1",
+    "mineflayer-tool": "^1.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^18.6.4",
+    "require-self": "^0.2.3",
+    "ts-standard": "^11.0.0",
+    "typescript": "^4.1.3"
+  },
+  "files": [
+    "lib/**/*"
+  ]
+}

metagpt/environment/mincraft_env/mineflayer/mineflayer-collectblock/src/BlockVeins.ts

@@ -0,0 +1,35 @@
+import { Bot } from 'mineflayer'
+import { Block } from 'prismarine-block'
+
+export function findFromVein (bot: Bot, block: Block, maxBlocks: number, maxDistance: number, floodRadius: number): Block[] {
+  const targets: Block[] = []
+  const open: Block[] = [block]
+  const type = block.type
+  const center = block.position
+
+  for (let i = 0; i < maxBlocks; i++) {
+    const next = open.pop()
+    if (next == null) break
+
+    targets.push(next)
+
+    for (let x = -floodRadius; x <= floodRadius; x++) {
+      for (let y = -floodRadius; y <= floodRadius; y++) {
+        for (let z = -floodRadius; z <= floodRadius; z++) {
+          const neighborPos = next.position.offset(x, y, z)
+          if (neighborPos.manhattanDistanceTo(center) > maxDistance) continue
+
+          const neighbor = bot.blockAt(neighborPos)
+          if (neighbor == null || neighbor.type !== type) continue
+
+          if (targets.includes(neighbor)) continue
+          if (open.includes(neighbor)) continue
+
+          open.push(neighbor)
+        }
+      }
+    }
+  }
+
+  return targets
+}