Merge branch 'geekan:main' into main

.devcontainer/README.md

@@ -0,0 +1,39 @@
+# Dev container
+
+This project includes a [dev container](https://containers.dev/), which lets you use a container as a full-featured dev environment.
+
+You can use the dev container configuration in this folder to build and start running MetaGPT locally! For more, refer to the main README under the home directory. 
+You can use it in [GitHub Codespaces](https://github.com/features/codespaces) or the [VS Code Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers).
+
+## GitHub Codespaces
+<a href="https://codespaces.new/geekan/MetaGPT"><img src="https://github.com/codespaces/badge.svg" alt="Open in GitHub Codespaces"></a>
+
+You may use the button above to open this repo in a Codespace
+
+For more info, check out the [GitHub documentation](https://docs.github.com/en/free-pro-team@latest/github/developing-online-with-codespaces/creating-a-codespace#creating-a-codespace).
+  
+## VS Code Dev Containers
+<a href="https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/geekan/MetaGPT"><img src="https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode" alt="Open in Dev Containers"></a>
+
+Note: If you click this link you will open the main repo and not your local cloned repo, you can use this link and replace with your username and cloned repo name: 
+https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/geekan/MetaGPT
+
+
+If you already have VS Code and Docker installed, you can use the button above to get started. This will cause VS Code to automatically install the Dev Containers extension if needed, clone the source code into a container volume, and spin up a dev container for use.
+
+You can also follow these steps to open this repo in a container using the VS Code Dev Containers extension:
+
+1. If this is your first time using a development container, please ensure your system meets the pre-reqs (i.e. have Docker installed) in the [getting started steps](https://aka.ms/vscode-remote/containers/getting-started).
+
+2. Open a locally cloned copy of the code:
+
+   - Fork and Clone this repository to your local filesystem.
+   - Press <kbd>F1</kbd> and select the **Dev Containers: Open Folder in Container...** command.
+   - Select the cloned copy of this folder, wait for the container to start, and try things out!
+
+You can learn more in the [Dev Containers documentation](https://code.visualstudio.com/docs/devcontainers/containers).
+
+## Tips and tricks
+
+* If you are working with the same repository folder in a container and Windows, you'll want consistent line endings (otherwise you may see hundreds of changes in the SCM view). The `.gitattributes` file in the root of this repo will disable line ending conversion and should prevent this. See [tips and tricks](https://code.visualstudio.com/docs/devcontainers/tips-and-tricks#_resolving-git-line-ending-issues-in-containers-resulting-in-many-modified-files) for more info.
+* If you'd like to review the contents of the image used in this dev container, you can check it out in the [devcontainers/images](https://github.com/devcontainers/images/tree/main/src/python) repo.

.devcontainer/devcontainer.json

@@ -0,0 +1,27 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Python 3",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/python:0-3.11",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			"settings": {},
+			"extensions": [
+				"streetsidesoftware.code-spell-checker"
+			]
+		}
+	},
+	
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "./.devcontainer/postCreateCommand.sh"
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.devcontainer/docker-compose.yaml

@@ -0,0 +1,31 @@
+version: '3'
+services:
+  metagpt:
+    build:
+      dockerfile: Dockerfile
+      context: ..
+    volumes:
+   # Update this to wherever you want VS Code to mount the folder of your project
+      - ..:/workspaces:cached
+    networks:
+      - metagpt-network 
+  #   environment:
+  #     MONGO_ROOT_USERNAME: root
+  #     MONGO_ROOT_PASSWORD: example123
+  #   depends_on:
+  #     - mongo   
+  # mongo:
+  #   image: mongo
+  #   restart: unless-stopped
+  #   environment:
+  #     MONGO_INITDB_ROOT_USERNAME: root
+  #     MONGO_INITDB_ROOT_PASSWORD: example123
+  #   ports:
+  #     - "27017:27017"
+  #   networks:
+  #     - metagpt-network
+
+networks:
+  metagpt-network:
+    driver: bridge
+    

.devcontainer/postCreateCommand.sh

@@ -0,0 +1,7 @@
+# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js.
+npm --version
+sudo npm install -g @mermaid-js/mermaid-cli
+
+# Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using:
+python --version
+python setup.py install

.dockerignore

@@ -0,0 +1,7 @@
+workspace
+tmp
+build
+workspace
+dist
+data
+geckodriver.log

README.md

@@ -19,6 +19,11 @@ # MetaGPT: The Multi-Agent Framework
 <a href="https://twitter.com/DeepWisdom2019"><img src="https://img.shields.io/twitter/follow/MetaGPT?style=social" alt="Twitter Follow"></a>
 </p>
 
+<p align="center">
+   <a href="https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/geekan/MetaGPT"><img src="https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode" alt="Open in Dev Containers"></a>
+   <a href="https://codespaces.new/geekan/MetaGPT"><img src="https://img.shields.io/badge/Github_Codespace-Open-blue?logo=github" alt="Open in GitHub Codespaces"></a>
+</p>
+
 1. MetaGPT takes a **one line requirement** as input and outputs **user stories / competitive analysis / requirements / data structures / APIs / documents, etc.**
 2. Internally, MetaGPT includes **product managers / architects / project managers / engineers.** It provides the entire process of a **software company along with carefully orchestrated SOPs.**
    1. `Code = SOP(Team)` is the core philosophy. We materialize SOP and apply it to teams composed of LLMs.
@@ -224,3 +229,9 @@ ## Contact Information
 ## Demo
 
 https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d
+
+## Join us
+📢 Join Our Discord Channel!
+https://discord.gg/4WdszVjv
+
+Looking forward to seeing you there! 🎉

config/config.yaml

@@ -25,12 +25,17 @@ RPM: 10
 
 #### for Search
 
+## Supported values: serpapi/google/serper/ddg
+#SEARCH_ENGINE: serpapi
+
 ## Visit https://serpapi.com/ to get key.
 #SERPAPI_API_KEY: "YOUR_API_KEY"
+
 ## Visit https://console.cloud.google.com/apis/credentials to get key.
 #GOOGLE_API_KEY: "YOUR_API_KEY"
 ## Visit https://programmablesearchengine.google.com/controlpanel/create to get id.
 #GOOGLE_CSE_ID: "YOUR_CSE_ID"
+
 ## Visit https://serper.dev/ to get key.
 #SERPER_API_KEY: "YOUR_API_KEY"
 

docs/README_CN.md

@@ -193,8 +193,8 @@ ## 演示
 
 https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d
 
-## 加入微信讨论群
+## 加入我们
 
-添加运营小姐姐，拉你入群
+📢 加入我们的Discord频道！https://discord.gg/4WdszVjv
 
-<img src="resources/20230811-214014.jpg" width = "30%" height = "30%" alt="MetaGPT WeChat Discuss Group" align=center />
+期待在那里与您相见！🎉

metagpt/actions/search_and_summarize.py

@@ -5,6 +5,8 @@
 @Author  : alexanderwu
 @File    : search_google.py
 """
+import pydantic
+
 from metagpt.actions import Action
 from metagpt.config import Config
 from metagpt.logs import logger
@@ -34,7 +36,7 @@ A: MLOps competitors
 8. Dataiku
 """
 
-SEARCH_AND_SUMMARIZE_SYSTEM_EN_US = SEARCH_AND_SUMMARIZE_SYSTEM.format(LANG='en-us')
+SEARCH_AND_SUMMARIZE_SYSTEM_EN_US = SEARCH_AND_SUMMARIZE_SYSTEM.format(LANG="en-us")
 
 SEARCH_AND_SUMMARIZE_PROMPT = """
 ### Reference Information
@@ -102,25 +104,26 @@ class SearchAndSummarize(Action):
     def __init__(self, name="", context=None, llm=None, engine=None, search_func=None):
         self.config = Config()
         self.engine = engine or self.config.search_engine
-        self.search_engine = SearchEngine(self.engine, run_func=search_func)
+
+        try:
+            self.search_engine = SearchEngine(self.engine, run_func=search_func)
+        except pydantic.ValidationError:
+            self.search_engine = None
+
         self.result = ""
         super().__init__(name, context, llm)
 
     async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str:
-        no_serpapi = not self.config.serpapi_api_key or 'YOUR_API_KEY' == self.config.serpapi_api_key
-        no_serper = not self.config.serper_api_key or 'YOUR_API_KEY' == self.config.serper_api_key
-        no_google = not self.config.google_api_key or 'YOUR_API_KEY' == self.config.google_api_key
-        
-        if no_serpapi and no_google and no_serper:
-            logger.warning('Configure one of SERPAPI_API_KEY, SERPER_API_KEY, GOOGLE_API_KEY to unlock full feature')
+        if self.search_engine is None:
+            logger.warning("Configure one of SERPAPI_API_KEY, SERPER_API_KEY, GOOGLE_API_KEY to unlock full feature")
             return ""
-        
+
         query = context[-1].content
         # logger.debug(query)
         rsp = await self.search_engine.run(query)
         self.result = rsp
         if not rsp:
-            logger.error('empty rsp...')
+            logger.error("empty rsp...")
             return ""
         # logger.info(rsp)
 
@@ -130,8 +133,8 @@ class SearchAndSummarize(Action):
             # PREFIX = self.prefix,
             ROLE=self.profile,
             CONTEXT=rsp,
-            QUERY_HISTORY='\n'.join([str(i) for i in context[:-1]]),
-            QUERY=str(context[-1])
+            QUERY_HISTORY="\n".join([str(i) for i in context[:-1]]),
+            QUERY=str(context[-1]),
         )
         result = await self._aask(prompt, system_prompt)
         logger.debug(prompt)

metagpt/config.py

@@ -45,16 +45,15 @@ class Config(metaclass=Singleton):
         self.global_proxy = self._get("GLOBAL_PROXY")
         self.openai_api_key = self._get("OPENAI_API_KEY")
         self.anthropic_api_key = self._get("Anthropic_API_KEY")
-        if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) \
-            and (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key):
+        if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and (
+            not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key
+        ):
             raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first")
         self.openai_api_base = self._get("OPENAI_API_BASE")
-        if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base:
-            openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy
-            if openai_proxy:
-                openai.proxy = openai_proxy
-            else:
-                logger.info("Set OPENAI_API_BASE in case of network issues")
+        openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy
+        if openai_proxy:
+            openai.proxy = openai_proxy
+            openai.api_base = self.openai_api_base
         self.openai_api_type = self._get("OPENAI_API_TYPE")
         self.openai_api_version = self._get("OPENAI_API_VERSION")
         self.openai_api_rpm = self._get("RPM", 3)
@@ -62,26 +61,25 @@ class Config(metaclass=Singleton):
         self.max_tokens_rsp = self._get("MAX_TOKENS", 2048)
         self.deployment_id = self._get("DEPLOYMENT_ID")
 
-        self.claude_api_key = self._get('Anthropic_API_KEY')
+        self.claude_api_key = self._get("Anthropic_API_KEY")
         self.serpapi_api_key = self._get("SERPAPI_API_KEY")
         self.serper_api_key = self._get("SERPER_API_KEY")
         self.google_api_key = self._get("GOOGLE_API_KEY")
         self.google_cse_id = self._get("GOOGLE_CSE_ID")
-        self.search_engine = self._get("SEARCH_ENGINE", SearchEngineType.SERPAPI_GOOGLE)
-
-        self.web_browser_engine = WebBrowserEngineType(self._get("WEB_BROWSER_ENGINE", "playwright"))
+        self.search_engine = SearchEngineType(self._get("SEARCH_ENGINE", SearchEngineType.SERPAPI_GOOGLE))
+        self.web_browser_engine = WebBrowserEngineType(self._get("WEB_BROWSER_ENGINE", WebBrowserEngineType.PLAYWRIGHT))
         self.playwright_browser_type = self._get("PLAYWRIGHT_BROWSER_TYPE", "chromium")
         self.selenium_browser_type = self._get("SELENIUM_BROWSER_TYPE", "chrome")
 
-        self.long_term_memory = self._get('LONG_TERM_MEMORY', False)
+        self.long_term_memory = self._get("LONG_TERM_MEMORY", False)
         if self.long_term_memory:
             logger.warning("LONG_TERM_MEMORY is True")
         self.max_budget = self._get("MAX_BUDGET", 10.0)
         self.total_cost = 0.0
 
-        self.puppeteer_config = self._get("PUPPETEER_CONFIG","")
-        self.mmdc = self._get("MMDC","mmdc")
-        self.calc_usage = self._get("CALC_USAGE",True)
+        self.puppeteer_config = self._get("PUPPETEER_CONFIG", "")
+        self.mmdc = self._get("MMDC", "mmdc")
+        self.calc_usage = self._get("CALC_USAGE", True)
         self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY")
         self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT")
 

metagpt/roles/researcher.py

@@ -84,10 +84,17 @@ class Researcher(Role):
         return msg
 
     def write_report(self, topic: str, content: str):
+        if not RESEARCH_PATH.exists():
+            RESEARCH_PATH.mkdir(parents=True)
         filepath = RESEARCH_PATH / f"{topic}.md"
         filepath.write_text(content)
 
 
 if __name__ == "__main__":
-    role = Researcher(language="en-us")
-    asyncio.run(role.run("dataiku vs. datarobot"))
+    import fire
+
+    async def main(topic: str, language="en-us"):
+        role = Researcher(topic, language=language)
+        await role.run(topic)
+
+    fire.Fire(main)

metagpt/tools/__init__.py

@@ -7,15 +7,15 @@
 """
 
 
-from enum import Enum, auto
+from enum import Enum
 
 
 class SearchEngineType(Enum):
-    SERPAPI_GOOGLE = auto()
-    DIRECT_GOOGLE = auto()
-    SERPER_GOOGLE = auto()
-    DUCK_DUCK_GO = auto()
-    CUSTOM_ENGINE = auto()
+    SERPAPI_GOOGLE = "serpapi"
+    SERPER_GOOGLE = "serper"
+    DIRECT_GOOGLE = "google"
+    DUCK_DUCK_GO = "ddg"
+    CUSTOM_ENGINE = "custom"
 
 
 class WebBrowserEngineType(Enum):

metagpt/tools/search_engine_ddg.py

@@ -7,11 +7,15 @@ import json
 from concurrent import futures
 from typing import Literal, overload
 
-from duckduckgo_search import DDGS
-from googleapiclient.errors import HttpError
+try:
+    from duckduckgo_search import DDGS
+except ImportError:
+    raise ImportError(
+        "To use this module, you should have the `duckduckgo_search` Python package installed. "
+        "You can install it by running the command: `pip install -e.[search-ddg]`"
+    )
 
 from metagpt.config import CONFIG
-from metagpt.logs import logger
 
 
 class DDGAPIWrapper:
@@ -19,6 +23,7 @@ class DDGAPIWrapper:
 
     To use this module, you should have the `duckduckgo_search` Python package installed.
     """
+
     def __init__(
         self,
         *,
@@ -77,15 +82,8 @@ class DDGAPIWrapper:
             query,
             max_results,
         )
-        try:
-            search_results = await future
-            # Extract the search result items from the response
+        search_results = await future
 
-        except HttpError as e:
-            # Handle errors in the API call
-            logger.exception(f"fail to search {query} for {e}")
-            search_results = []
-        
         # Return the list of search result URLs
         if as_string:
             return json.dumps(search_results, ensure_ascii=False)
@@ -93,11 +91,8 @@ class DDGAPIWrapper:
 
     def _search_from_ddgs(self, query: str, max_results: int):
         return [
-            {
-                "link": i["href"],
-                "snippet": i["body"],
-                "title": i["title"]
-            } for (_, i) in zip(range(max_results), self.ddgs.text(query))
+            {"link": i["href"], "snippet": i["body"], "title": i["title"]}
+            for (_, i) in zip(range(max_results), self.ddgs.text(query))
         ]
 
 

metagpt/tools/search_engine_googleapi.py

@@ -5,30 +5,61 @@ from __future__ import annotations
 import asyncio
 import json
 from concurrent import futures
+from typing import Optional
 from urllib.parse import urlparse
 
 import httplib2
-from googleapiclient.discovery import build
-from googleapiclient.errors import HttpError
+from pydantic import BaseModel, validator
 
 from metagpt.config import CONFIG
 from metagpt.logs import logger
 
+try:
+    from googleapiclient.discovery import build
+    from googleapiclient.errors import HttpError
+except ImportError:
+    raise ImportError(
+        "To use this module, you should have the `google-api-python-client` Python package installed. "
+        "You can install it by running the command: `pip install -e.[search-google]`"
+    )
 
-class GoogleAPIWrapper:
-    """Wrapper around GoogleAPI.
 
-    To use this module, you should have the `google-api-python-client` Python package installed
-    and set property values for the configurations `GOOGLE_API_KEY` and `GOOGLE_CSE_ID`. See 
-    https://programmablesearchengine.google.com/controlpanel/all.
-    """
-    def __init__(
-        self,
-        *,
-        loop: asyncio.AbstractEventLoop | None = None,
-        executor: futures.Executor | None = None,
-    ):
-        build_kwargs = {"developerKey": CONFIG.google_api_key}
+class GoogleAPIWrapper(BaseModel):
+    google_api_key: Optional[str] = None
+    google_cse_id: Optional[str] = None
+    loop: Optional[asyncio.AbstractEventLoop] = None
+    executor: Optional[futures.Executor] = None
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    @validator("google_api_key", always=True)
+    @classmethod
+    def check_google_api_key(cls, val: str):
+        val = val or CONFIG.google_api_key
+        if not val:
+            raise ValueError(
+                "To use, make sure you provide the google_api_key when constructing an object. Alternatively, "
+                "ensure that the environment variable GOOGLE_API_KEY is set with your API key. You can obtain "
+                "an API key from https://console.cloud.google.com/apis/credentials."
+            )
+        return val
+
+    @validator("google_cse_id", always=True)
+    @classmethod
+    def check_google_cse_id(cls, val: str):
+        val = val or CONFIG.google_cse_id
+        if not val:
+            raise ValueError(
+                "To use, make sure you provide the google_cse_id when constructing an object. Alternatively, "
+                "ensure that the environment variable GOOGLE_CSE_ID is set with your API key. You can obtain "
+                "an API key from https://programmablesearchengine.google.com/controlpanel/create."
+            )
+        return val
+
+    @property
+    def google_api_client(self):
+        build_kwargs = {"developerKey": self.google_api_key}
         if CONFIG.global_proxy:
             parse_result = urlparse(CONFIG.global_proxy)
             proxy_type = parse_result.scheme
@@ -42,10 +73,7 @@ class GoogleAPIWrapper:
                 ),
             )
         service = build("customsearch", "v1", **build_kwargs)
-        self.google_api_client = service.cse()
-        self.custom_search_engine_id = CONFIG.google_cse_id
-        self.loop = loop
-        self.executor = executor
+        return service.cse()
 
     async def run(
         self,
@@ -69,12 +97,7 @@ class GoogleAPIWrapper:
         """
         loop = self.loop or asyncio.get_event_loop()
         future = loop.run_in_executor(
-            self.executor,
-            self.google_api_client.list(
-                q=query,
-                num=max_results,
-                cx=self.custom_search_engine_id
-            ).execute
+            self.executor, self.google_api_client.list(q=query, num=max_results, cx=self.google_cse_id).execute
         )
         try:
             result = await future
@@ -85,13 +108,13 @@ class GoogleAPIWrapper:
             # Handle errors in the API call
             logger.exception(f"fail to search {query} for {e}")
             search_results = []
-        
+
         focus = focus or ["snippet", "link", "title"]
         details = [{i: j for i, j in item_dict.items() if i in focus} for item_dict in search_results]
         # Return the list of search result URLs
         if as_string:
             return safe_google_results(details)
-        
+
         return details
 
 

metagpt/tools/search_engine_serpapi.py

@@ -8,19 +8,12 @@
 from typing import Any, Dict, Optional, Tuple
 
 import aiohttp
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, validator
 
-from metagpt.config import Config
+from metagpt.config import CONFIG
 
 
 class SerpAPIWrapper(BaseModel):
-    """Wrapper around SerpAPI.
-
-    To use, you should have the ``google-search-results`` python package installed,
-    and the environment variable ``SERPAPI_API_KEY`` set with your API key, or pass
-    `serpapi_api_key` as a named parameter to the constructor.
-    """
-
     search_engine: Any  #: :meta private:
     params: dict = Field(
         default={
@@ -30,14 +23,25 @@ class SerpAPIWrapper(BaseModel):
             "hl": "en",
         }
     )
-    config = Config()
-    serpapi_api_key: Optional[str] = config.serpapi_api_key
+    serpapi_api_key: Optional[str] = None
     aiosession: Optional[aiohttp.ClientSession] = None
 
     class Config:
         arbitrary_types_allowed = True
 
-    async def run(self, query: str, max_results: int = 8, as_string: bool = True, **kwargs: Any) -> str:
+    @validator("serpapi_api_key", always=True)
+    @classmethod
+    def check_serpapi_api_key(cls, val: str):
+        val = val or CONFIG.serpapi_api_key
+        if not val:
+            raise ValueError(
+                "To use, make sure you provide the serpapi_api_key when constructing an object. Alternatively, "
+                "ensure that the environment variable SERPAPI_API_KEY is set with your API key. You can obtain "
+                "an API key from https://serpapi.com/."
+            )
+        return val
+
+    async def run(self, query, max_results: int = 8, as_string: bool = True, **kwargs: Any) -> str:
         """Run query through SerpAPI and parse result async."""
         return self._process_response(await self.results(query, max_results), as_string=as_string)
 
@@ -48,8 +52,6 @@ class SerpAPIWrapper(BaseModel):
             params = self.get_params(query)
             params["source"] = "python"
             params["num"] = max_results
-            if self.serpapi_api_key:
-                params["serp_api_key"] = self.serpapi_api_key
             params["output"] = "json"
             url = "https://serpapi.com/search"
             return url, params
@@ -104,7 +106,7 @@ class SerpAPIWrapper(BaseModel):
         if res.get("organic_results"):
             toret_l += [get_focused(i) for i in res.get("organic_results")]
 
-        return str(toret) + '\n' + str(toret_l) if as_string else toret_l
+        return str(toret) + "\n" + str(toret_l) if as_string else toret_l
 
 
 if __name__ == "__main__":

metagpt/tools/search_engine_serper.py

@@ -9,33 +9,32 @@ import json
 from typing import Any, Dict, Optional, Tuple
 
 import aiohttp
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, validator
 
-from metagpt.config import Config
+from metagpt.config import CONFIG
 
 
 class SerperWrapper(BaseModel):
-    """Wrapper around SerpAPI.
-
-    To use, you should have the ``google-search-results`` python package installed,
-    and the environment variable ``SERPAPI_API_KEY`` set with your API key, or pass
-    `serpapi_api_key` as a named parameter to the constructor.
-    """
-
     search_engine: Any  #: :meta private:
-    payload: dict = Field(
-        default={
-            "page": 1,
-            "num": 10
-        }
-    )
-    config = Config()
-    serper_api_key: Optional[str] = config.serper_api_key
+    payload: dict = Field(default={"page": 1, "num": 10})
+    serper_api_key: Optional[str] = None
     aiosession: Optional[aiohttp.ClientSession] = None
 
     class Config:
         arbitrary_types_allowed = True
 
+    @validator("serper_api_key", always=True)
+    @classmethod
+    def check_serper_api_key(cls, val: str):
+        val = val or CONFIG.serper_api_key
+        if not val:
+            raise ValueError(
+                "To use, make sure you provide the serper_api_key when constructing an object. Alternatively, "
+                "ensure that the environment variable SERPER_API_KEY is set with your API key. You can obtain "
+                "an API key from https://serper.dev/."
+            )
+        return val
+
     async def run(self, query: str, max_results: int = 8, as_string: bool = True, **kwargs: Any) -> str:
         """Run query through Serper and parse result async."""
         if isinstance(query, str):
@@ -76,18 +75,17 @@ class SerperWrapper(BaseModel):
         return json.dumps(payloads, sort_keys=True)
 
     def get_headers(self) -> Dict[str, str]:
-        headers = {
-            'X-API-KEY':  self.serper_api_key,
-            'Content-Type': 'application/json'
-        }
+        headers = {"X-API-KEY": self.serper_api_key, "Content-Type": "application/json"}
         return headers
 
     @staticmethod
     def _process_response(res: dict, as_string: bool = False) -> str:
         """Process response from SerpAPI."""
         # logger.debug(res)
-        focus = ['title', 'snippet', 'link']
-        def get_focused(x): return {i: j for i, j in x.items() if i in focus}
+        focus = ["title", "snippet", "link"]
+
+        def get_focused(x):
+            return {i: j for i, j in x.items() if i in focus}
 
         if "error" in res.keys():
             raise ValueError(f"Got error from SerpAPI: {res['error']}")
@@ -95,20 +93,11 @@ class SerperWrapper(BaseModel):
             toret = res["answer_box"]["answer"]
         elif "answer_box" in res.keys() and "snippet" in res["answer_box"].keys():
             toret = res["answer_box"]["snippet"]
-        elif (
-            "answer_box" in res.keys()
-            and "snippet_highlighted_words" in res["answer_box"].keys()
-        ):
+        elif "answer_box" in res.keys() and "snippet_highlighted_words" in res["answer_box"].keys():
             toret = res["answer_box"]["snippet_highlighted_words"][0]
-        elif (
-            "sports_results" in res.keys()
-            and "game_spotlight" in res["sports_results"].keys()
-        ):
+        elif "sports_results" in res.keys() and "game_spotlight" in res["sports_results"].keys():
             toret = res["sports_results"]["game_spotlight"]
-        elif (
-            "knowledge_graph" in res.keys()
-            and "description" in res["knowledge_graph"].keys()
-        ):
+        elif "knowledge_graph" in res.keys() and "description" in res["knowledge_graph"].keys():
             toret = res["knowledge_graph"]["description"]
         elif "snippet" in res["organic"][0].keys():
             toret = res["organic"][0]["snippet"]
@@ -121,7 +110,7 @@ class SerperWrapper(BaseModel):
         if res.get("organic"):
             toret_l += [get_focused(i) for i in res.get("organic")]
 
-        return str(toret) + '\n' + str(toret_l) if as_string else toret_l
+        return str(toret) + "\n" + str(toret_l) if as_string else toret_l
 
 
 if __name__ == "__main__":

metagpt/utils/token_counter.py

@@ -96,7 +96,7 @@ def count_string_tokens(string: str, model_name: str) -> int:
     return len(encoding.encode(string))
 
 
-def get_max_completion_tokens(messages: list[dict], model: str, default: int) -> int: 
+def get_max_completion_tokens(messages: list[dict], model: str, default: int) -> int:
     """Calculate the maximum number of completion tokens for a given model and list of messages.
 
     Args:
@@ -108,4 +108,4 @@ def get_max_completion_tokens(messages: list[dict], model: str, default: int) ->
     """
     if model not in TOKEN_MAX:
         return default
-    return TOKEN_MAX[model] - count_message_tokens(messages)
+    return TOKEN_MAX[model] - count_message_tokens(messages) - 1

requirements.txt

@@ -4,7 +4,6 @@ channels==4.0.0
 # chromadb==0.3.22
 # Django==4.1.5
 # docx==0.2.4
-duckduckgo_search==2.9.4
 #faiss==1.5.3
 faiss_cpu==1.7.4
 fire==0.4.0

setup.py

@@ -45,6 +45,8 @@ setup(
     extras_require={
         "playwright": ["playwright>=1.26", "beautifulsoup4"],
         "selenium": ["selenium>4", "webdriver_manager", "beautifulsoup4"],
+        "search-google": ["google-api-python-client==2.94.0"],
+        "search-ddg": ["duckduckgo-search==3.8.5"],
     },
     cmdclass={
         "install_mermaid": InstallMermaidCLI,

startup.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import asyncio
-
+import platform
 import fire
 
 from metagpt.roles import Architect, Engineer, ProductManager, ProjectManager, QaEngineer
@@ -33,6 +33,8 @@ def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool
     :param code_review: Whether to use code review.
     :return:
     """
+    if platform.system() == "Windows":
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
     asyncio.run(startup(idea, investment, n_round, code_review, run_tests))