Merge branch 'main' into ci_dev

README.md

@@ -62,7 +62,27 @@ ### Pip installation
 
 ```bash
 pip install metagpt
-metagpt --init-config  # create ~/.metagpt/config2.yaml, modify it to your own config
+# https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html
+metagpt --init-config  # it will create ~/.metagpt/config2.yaml, just modify it to your needs
+```
+
+### Configuration
+
+You can configure `~/.metagpt/config2.yaml` according to the [example](https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml) and [doc](https://docs.deepwisdom.ai/main/en/guide/get_started/configuration.html):
+
+```yaml
+llm:
+  api_type: "openai"  # or azure / ollama / open_llm etc. Check LLMType for more options
+  model: "gpt-4-turbo-preview"  # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
+  base_url: "https://api.openai.com/v1"  # or forward url / other llm url
+  api_key: "YOUR_API_KEY"
+```
+
+### Usage
+
+After installation, you can use it as CLI
+
+```bash
 metagpt "Create a 2048 game"  # this will create a repo in ./workspace
 ```
 
@@ -75,27 +95,7 @@ ### Pip installation
 ```
 
 detail installation please refer to [cli_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-stable-version)
-
-### Docker installation
-> Note: In the Windows, you need to replace "/opt/metagpt" with a directory that Docker has permission to create, such as "D:\Users\x\metagpt"
-
-```bash
-# Step 1: Download metagpt official image and prepare config2.yaml
-docker pull metagpt/metagpt:latest
-mkdir -p /opt/metagpt/{config,workspace}
-docker run --rm metagpt/metagpt:latest cat /app/metagpt/config/config2.yaml > /opt/metagpt/config/config2.yaml
-vim /opt/metagpt/config/config2.yaml # Change the config
-
-# Step 2: Run metagpt demo with container
-docker run --rm \
-    --privileged \
-    -v /opt/metagpt/config/config2.yaml:/app/metagpt/config/config2.yaml \
-    -v /opt/metagpt/workspace:/app/metagpt/workspace \
-    metagpt/metagpt:latest \
-    metagpt "Create a 2048 game"
-```
-
-detail installation please refer to [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)
+ or [docker_install](https://docs.deepwisdom.ai/main/en/guide/get_started/installation.html#install-with-docker)
 
 ### QuickStart & Demo Video
 - Try it on [MetaGPT Huggingface Space](https://huggingface.co/spaces/deepwisdom/MetaGPT)

config/config2.example.yaml

@@ -1,10 +1,12 @@
 llm:
-  api_type: "openai"  # or azure / ollama etc.
+  api_type: "openai"  # or azure / ollama / open_llm etc. Check LLMType for more options
   base_url: "YOUR_BASE_URL"
   api_key: "YOUR_API_KEY"
   model: "gpt-4-turbo-preview"  # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
+  repair_llm_output: true  # when the output is not a valid json, try to repair it
+  proxy: "YOUR_PROXY"  # for LLM API requests
 
-proxy: "YOUR_PROXY"
+proxy: "YOUR_PROXY"  # for tools like requests, playwright, selenium, etc.
 
 search:
   api_type: "google"
@@ -41,5 +43,3 @@ iflytek_api_key: "YOUR_API_KEY"
 iflytek_api_secret: "YOUR_API_SECRET"
 
 metagpt_tti_url: "YOUR_MODEL_URL"
-
-repair_llm_output: true

config/config2.yaml

@@ -1,3 +1,7 @@
+# Full Example: https://github.com/geekan/MetaGPT/blob/main/config/config2.example.yaml
+# Reflected Code: https://github.com/geekan/MetaGPT/blob/main/metagpt/config2.py
 llm:
-  api_key: "YOUR_API_KEY"
-  model: "gpt-4-turbo-preview"  # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
+  api_type: "openai"  # or azure / ollama / open_llm etc. Check LLMType for more options
+  model: "gpt-4-turbo-preview"  # or gpt-3.5-turbo-1106 / gpt-4-1106-preview
+  base_url: "https://api.openai.com/v1"  # or forward url / other llm url
+  api_key: "YOUR_API_KEY"

docs/scripts/coverage.sh

@@ -1 +1 @@
-coverage run --source ./metagpt -m pytest --durations=0 --timeout=100 && coverage report -m && coverage html && open htmlcov/index.html
+coverage run --source ./metagpt -m pytest -n 8 --durations=0 --timeout=100 && coverage report -m && coverage html && open htmlcov/index.html

tests/conftest.py

@@ -14,6 +14,7 @@ import re
 import uuid
 from typing import Callable
 
+import aiohttp.web
 import pytest
 
 from metagpt.const import DEFAULT_WORKSPACE_ROOT, TEST_DATA_PATH
@@ -171,9 +172,8 @@ def new_filename(mocker):
     yield mocker
 
 
-@pytest.fixture(scope="session")
-def search_rsp_cache():
-    rsp_cache_file_path = TEST_DATA_PATH / "search_rsp_cache.json"  # read repo-provided
+def _rsp_cache(name):
+    rsp_cache_file_path = TEST_DATA_PATH / f"{name}.json"  # read repo-provided
     if os.path.exists(rsp_cache_file_path):
         with open(rsp_cache_file_path, "r") as f1:
             rsp_cache_json = json.load(f1)
@@ -184,6 +184,16 @@ def search_rsp_cache():
         json.dump(rsp_cache_json, f2, indent=4, ensure_ascii=False)
 
 
+@pytest.fixture(scope="session")
+def search_rsp_cache():
+    yield from _rsp_cache("search_rsp_cache")
+
+
+@pytest.fixture(scope="session")
+def mermaid_rsp_cache():
+    yield from _rsp_cache("mermaid_rsp_cache")
+
+
 @pytest.fixture
 def aiohttp_mocker(mocker):
     MockResponse = type("MockResponse", (MockAioResponse,), {})
@@ -231,3 +241,32 @@ def search_engine_mocker(aiohttp_mocker, curl_cffi_mocker, httplib2_mocker, sear
     aiohttp_mocker.rsp_cache = httplib2_mocker.rsp_cache = curl_cffi_mocker.rsp_cache = search_rsp_cache
     aiohttp_mocker.check_funcs = httplib2_mocker.check_funcs = curl_cffi_mocker.check_funcs = check_funcs
     yield check_funcs
+
+
+@pytest.fixture
+def http_server():
+    async def handler(request):
+        return aiohttp.web.Response(
+            text="""<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8">
+            <title>MetaGPT</title></head><body><h1>MetaGPT</h1></body></html>""",
+            content_type="text/html",
+        )
+
+    async def start():
+        server = aiohttp.web.Server(handler)
+        runner = aiohttp.web.ServerRunner(server)
+        await runner.setup()
+        site = aiohttp.web.TCPSite(runner, "localhost", 0)
+        await site.start()
+        host, port = site._server.sockets[0].getsockname()
+        return site, f"http://{host}:{port}"
+
+    return start
+
+
+@pytest.fixture
+def mermaid_mocker(aiohttp_mocker, mermaid_rsp_cache):
+    check_funcs: dict[tuple[str, str], Callable[[dict], str]] = {}
+    aiohttp_mocker.rsp_cache = mermaid_rsp_cache
+    aiohttp_mocker.check_funcs = check_funcs
+    yield check_funcs

tests/metagpt/utils/test_mermaid.py

@@ -14,7 +14,7 @@ from metagpt.utils.mermaid import MMC1, mermaid_to_file
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("engine", ["nodejs", "ink"])  # TODO: playwright and pyppeteer
-async def test_mermaid(engine, context):
+async def test_mermaid(engine, context, mermaid_mocker):
     # nodejs prerequisites: npm install -g @mermaid-js/mermaid-cli
     # ink prerequisites: connected to internet
     # playwright prerequisites: playwright install --with-deps chromium

tests/mock/mock_aiohttp.py

@@ -10,6 +10,7 @@ class MockAioResponse:
     check_funcs: dict[tuple[str, str], Callable[[dict], str]] = {}
     rsp_cache: dict[str, str] = {}
     name = "aiohttp"
+    status = 200
 
     def __init__(self, session, method, url, **kwargs) -> None:
         fn = self.check_funcs.get((method, url))
@@ -22,6 +23,7 @@ class MockAioResponse:
     async def __aenter__(self):
         if self.response:
             await self.response.__aenter__()
+            self.status = self.response.status
         elif self.mng:
             self.response = await self.mng.__aenter__()
         return self
@@ -41,6 +43,17 @@ class MockAioResponse:
         self.rsp_cache[self.key] = data
         return data
 
+    @property
+    def content(self):
+        return self
+
+    async def read(self):
+        if self.key in self.rsp_cache:
+            return eval(self.rsp_cache[self.key])
+        data = await self.response.content.read()
+        self.rsp_cache[self.key] = str(data)
+        return data
+
     def raise_for_status(self):
         if self.response:
             self.response.raise_for_status()