From 8c94195c0b6ec48daab98f042d7cbac6ff22aef2 Mon Sep 17 00:00:00 2001 From: Chao Lan Date: Sun, 6 Aug 2023 05:50:15 +0800 Subject: [PATCH 01/22] fix the bug can not use proxy --- metagpt/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index d53571468..e92c4fa75 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -49,11 +49,11 @@ class Config(metaclass=Singleton): self.openai_api_base = self._get("OPENAI_API_BASE") if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base: + logger.info("Set OPENAI_API_BASE in case of network issues") + else: openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy if openai_proxy: openai.proxy = openai_proxy - else: - logger.info("Set OPENAI_API_BASE in case of network issues") self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) From c1c73a235757c0fa9d6c88b55575df9b1656be58 Mon Sep 17 00:00:00 2001 From: Chao Lan Date: Mon, 7 Aug 2023 16:26:31 +0800 Subject: [PATCH 02/22] no longer checking the value of OPENAI_API_BASE --- metagpt/config.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index e92c4fa75..821ae7cd0 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -48,12 +48,9 @@ class Config(metaclass=Singleton): raise NotConfiguredException("Set OPENAI_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") - if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base: - logger.info("Set OPENAI_API_BASE in case of network issues") - else: - openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy - if openai_proxy: - openai.proxy = openai_proxy + openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy + if openai_proxy: + openai.proxy = openai_proxy self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) From 848eb373419ec667e5cf257f787e59cb20a495d2 Mon Sep 17 00:00:00 2001 From: Chao Lan Date: Wed, 9 Aug 2023 03:24:51 +0000 Subject: [PATCH 03/22] set OPENAI_API_BASE when using proxy --- metagpt/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/config.py b/metagpt/config.py index 821ae7cd0..48010dcec 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -51,6 +51,7 @@ class Config(metaclass=Singleton): openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy if openai_proxy: openai.proxy = openai_proxy + openai.api_base = self.openai_api_base self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) From bc7e9687c8a35e52a46dcfb42c91eb06328b6cc5 Mon Sep 17 00:00:00 2001 From: leonzh0u Date: Wed, 9 Aug 2023 20:33:25 -0400 Subject: [PATCH 04/22] try github codespace --- .devcontainer/devcontainer.json | 27 +++++++++++++++++++++++++++ .devcontainer/postCreateCommand.sh | 7 +++++++ 2 files changed, 34 insertions(+) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/postCreateCommand.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..a774d0ed1 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,27 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "Python 3", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/python:0-3.11", + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Configure tool-specific properties. + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + "settings": {}, + "extensions": [ + "streetsidesoftware.code-spell-checker" + ] + } + }, + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "./.devcontainer/postCreateCommand.sh" + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh new file mode 100644 index 000000000..06d12e408 --- /dev/null +++ b/.devcontainer/postCreateCommand.sh @@ -0,0 +1,7 @@ +# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. +npm --version +sudo npm install -g @mermaid-js/mermaid-cli + +# Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using: +python --version +python setup.py install \ No newline at end of file From 23c88a72ff8c19b63a83025a9ee6516456bd12c8 Mon Sep 17 00:00:00 2001 From: Leon Zhou Date: Wed, 9 Aug 2023 20:45:26 -0400 Subject: [PATCH 05/22] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7eaaa2f69..4fd179f53 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ # MetaGPT: The Multi-Agent Framework roadmap Twitter Follow

+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/geekan/MetaGPT) 1. MetaGPT takes a **one line requirement** as input and outputs **user stories / competitive analysis / requirements / data structures / APIs / documents, etc.** 2. Internally, MetaGPT includes **product managers / architects / project managers / engineers.** It provides the entire process of a **software company along with carefully orchestrated SOPs.** From 493c2a0b8609c79f7f64e8314b24c6861d760939 Mon Sep 17 00:00:00 2001 From: Leon Zhou Date: Wed, 9 Aug 2023 20:47:51 -0400 Subject: [PATCH 06/22] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4fd179f53..d59513fd3 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ # MetaGPT: The Multi-Agent Framework roadmap Twitter Follow

+ [![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/geekan/MetaGPT) 1. MetaGPT takes a **one line requirement** as input and outputs **user stories / competitive analysis / requirements / data structures / APIs / documents, etc.** From f4c8311b4c56d99ac6d96c9a00468992eb5c7d24 Mon Sep 17 00:00:00 2001 From: Leon Zhou Date: Wed, 9 Aug 2023 20:54:53 -0400 Subject: [PATCH 07/22] Update README.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d59513fd3..370202998 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,10 @@ # MetaGPT: The Multi-Agent Framework Twitter Follow

-[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/geekan/MetaGPT) +

+ Open in Dev Containers + Open in GitHub Codespaces +

1. MetaGPT takes a **one line requirement** as input and outputs **user stories / competitive analysis / requirements / data structures / APIs / documents, etc.** 2. Internally, MetaGPT includes **product managers / architects / project managers / engineers.** It provides the entire process of a **software company along with carefully orchestrated SOPs.** From 36f6aaed0eb5817bfeb45eb39fbf8ecdc0db8f62 Mon Sep 17 00:00:00 2001 From: Leon Zhou Date: Wed, 9 Aug 2023 21:09:58 -0400 Subject: [PATCH 08/22] Create docker-compose.yaml --- .devcontainer/docker-compose.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .devcontainer/docker-compose.yaml diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml new file mode 100644 index 000000000..a9988b1f3 --- /dev/null +++ b/.devcontainer/docker-compose.yaml @@ -0,0 +1,31 @@ +version: '3' +services: + metagpt: + build: + dockerfile: Dockerfile + context: .. + volumes: + # Update this to wherever you want VS Code to mount the folder of your project + - ..:/workspaces:cached + networks: + - metagpt-network + # environment: + # MONGO_ROOT_USERNAME: root + # MONGO_ROOT_PASSWORD: example123 + # depends_on: + # - mongo + # mongo: + # image: mongo + # restart: unless-stopped + # environment: + # MONGO_INITDB_ROOT_USERNAME: root + # MONGO_INITDB_ROOT_PASSWORD: example123 + # ports: + # - "27017:27017" + # networks: + # - metagpt-network + +networks: + metagpt-network: + driver: bridge + From e24e58f3142b8e5de3abc5c1e1223d40c5e3a03c Mon Sep 17 00:00:00 2001 From: Leon Zhou Date: Wed, 9 Aug 2023 21:11:25 -0400 Subject: [PATCH 09/22] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 370202998..4ed684118 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ # MetaGPT: The Multi-Agent Framework

Open in Dev Containers - Open in GitHub Codespaces + Open in GitHub Codespaces

1. MetaGPT takes a **one line requirement** as input and outputs **user stories / competitive analysis / requirements / data structures / APIs / documents, etc.** From b995d832730650d14738716c3a57d35465aee8c5 Mon Sep 17 00:00:00 2001 From: Leon Zhou Date: Wed, 9 Aug 2023 21:18:26 -0400 Subject: [PATCH 10/22] Create README.md --- .devcontainer/README.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .devcontainer/README.md diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 000000000..47ae0ddd5 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,38 @@ +# Dev container + +This project includes a [dev container](https://containers.dev/), which lets you use a container as a full-featured dev environment. + +You can use the dev container configuration in this folder to build and run the app without needing to install any of its tools locally! You can use it in [GitHub Codespaces](https://github.com/features/codespaces) or the [VS Code Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). + +## GitHub Codespaces +Open in GitHub Codespaces + +You may use the button above to open this repo in a Codespace + +For more info, check out the [GitHub documentation](https://docs.github.com/en/free-pro-team@latest/github/developing-online-with-codespaces/creating-a-codespace#creating-a-codespace). + +## VS Code Dev Containers +Open in Dev Containers + +Note: If you click this link you will open the main repo and not your local cloned repo, you can use this link and replace with your username and cloned repo name: +https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com// + + +If you already have VS Code and Docker installed, you can use the button above to get started. This will cause VS Code to automatically install the Dev Containers extension if needed, clone the source code into a container volume, and spin up a dev container for use. + +You can also follow these steps to open this repo in a container using the VS Code Dev Containers extension: + +1. If this is your first time using a development container, please ensure your system meets the pre-reqs (i.e. have Docker installed) in the [getting started steps](https://aka.ms/vscode-remote/containers/getting-started). + +2. Open a locally cloned copy of the code: + + - Fork and Clone this repository to your local filesystem. + - Press F1 and select the **Dev Containers: Open Folder in Container...** command. + - Select the cloned copy of this folder, wait for the container to start, and try things out! + +You can learn more in the [Dev Containers documentation](https://code.visualstudio.com/docs/devcontainers/containers). + +## Tips and tricks + +* If you are working with the same repository folder in a container and Windows, you'll want consistent line endings (otherwise you may see hundreds of changes in the SCM view). The `.gitattributes` file in the root of this repo will disable line ending conversion and should prevent this. See [tips and tricks](https://code.visualstudio.com/docs/devcontainers/tips-and-tricks#_resolving-git-line-ending-issues-in-containers-resulting-in-many-modified-files) for more info. +* If you'd like to review the contents of the image used in this dev container, you can check it out in the [devcontainers/images](https://github.com/devcontainers/images/tree/main/src/python) repo. From d3dcd55de4e580f1b3b79c14622b53dae7b2240a Mon Sep 17 00:00:00 2001 From: Leon Zhou Date: Sat, 12 Aug 2023 00:12:00 -0400 Subject: [PATCH 11/22] Update README.md --- .devcontainer/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 47ae0ddd5..4bc6012bf 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -2,7 +2,8 @@ # Dev container This project includes a [dev container](https://containers.dev/), which lets you use a container as a full-featured dev environment. -You can use the dev container configuration in this folder to build and run the app without needing to install any of its tools locally! You can use it in [GitHub Codespaces](https://github.com/features/codespaces) or the [VS Code Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). +You can use the dev container configuration in this folder to build and start running MetaGPT locally! For more, refer to the main README under the home directory. +You can use it in [GitHub Codespaces](https://github.com/features/codespaces) or the [VS Code Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). ## GitHub Codespaces Open in GitHub Codespaces From 95b317b329cc8903ba032ba49e53601197a2cf7b Mon Sep 17 00:00:00 2001 From: Leon Zhou Date: Sat, 12 Aug 2023 00:14:48 -0400 Subject: [PATCH 12/22] Update README.md --- .devcontainer/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 4bc6012bf..dd088aab1 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -16,7 +16,7 @@ ## VS Code Dev Containers Open in Dev Containers Note: If you click this link you will open the main repo and not your local cloned repo, you can use this link and replace with your username and cloned repo name: -https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com// +https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/geekan/MetaGPT If you already have VS Code and Docker installed, you can use the button above to get started. This will cause VS Code to automatically install the Dev Containers extension if needed, clone the source code into a container volume, and spin up a dev container for use. From c43446516885cc978adf64d1a85fefd340149566 Mon Sep 17 00:00:00 2001 From: Leon Zhou Date: Sat, 12 Aug 2023 00:26:52 -0400 Subject: [PATCH 13/22] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ed684118..83536bbea 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ # MetaGPT: The Multi-Agent Framework

Open in Dev Containers - Open in GitHub Codespaces + Open in GitHub Codespaces

1. MetaGPT takes a **one line requirement** as input and outputs **user stories / competitive analysis / requirements / data structures / APIs / documents, etc.** From 7e329a478afcedfe6a353cf5fb5cec1863fb9f83 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Thu, 17 Aug 2023 17:14:43 +0800 Subject: [PATCH 14/22] Make the report output directory if it doesn't exist. --- metagpt/roles/researcher.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 815cfa172..acb46c718 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -84,10 +84,17 @@ class Researcher(Role): return msg def write_report(self, topic: str, content: str): + if not RESEARCH_PATH.exists(): + RESEARCH_PATH.mkdir(parents=True) filepath = RESEARCH_PATH / f"{topic}.md" filepath.write_text(content) if __name__ == "__main__": - role = Researcher(language="en-us") - asyncio.run(role.run("dataiku vs. datarobot")) + import fire + + async def main(topic: str, language="en-us"): + role = Researcher(topic, language=language) + await role.run(topic) + + fire.Fire(main) From 6e6e91660db88230fda8a25f947fc36fb6c14d28 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Thu, 17 Aug 2023 17:37:20 +0800 Subject: [PATCH 15/22] Make the SearchEngine more user-friendly. --- .dockerignore | 7 +++ config/config.yaml | 5 ++ metagpt/config.py | 20 +++--- metagpt/tools/__init__.py | 12 ++-- metagpt/tools/search_engine_ddg.py | 27 ++++----- metagpt/tools/search_engine_googleapi.py | 77 +++++++++++++++--------- metagpt/tools/search_engine_serpapi.py | 32 +++++----- metagpt/tools/search_engine_serper.py | 61 ++++++++----------- requirements.txt | 1 - setup.py | 2 + 10 files changed, 133 insertions(+), 111 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..2968dd34d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +workspace +tmp +build +workspace +dist +data +geckodriver.log diff --git a/config/config.yaml b/config/config.yaml index 303f4824b..274cdf469 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -25,12 +25,17 @@ RPM: 10 #### for Search +## Supported values: serpapi/google/serper/ddg +#SEARCH_ENGINE: serpapi + ## Visit https://serpapi.com/ to get key. #SERPAPI_API_KEY: "YOUR_API_KEY" + ## Visit https://console.cloud.google.com/apis/credentials to get key. #GOOGLE_API_KEY: "YOUR_API_KEY" ## Visit https://programmablesearchengine.google.com/controlpanel/create to get id. #GOOGLE_CSE_ID: "YOUR_CSE_ID" + ## Visit https://serper.dev/ to get key. #SERPER_API_KEY: "YOUR_API_KEY" diff --git a/metagpt/config.py b/metagpt/config.py index d7339caf5..21f180455 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -45,8 +45,9 @@ class Config(metaclass=Singleton): self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") - if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) \ - and (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key): + if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base: @@ -62,26 +63,25 @@ class Config(metaclass=Singleton): self.max_tokens_rsp = self._get("MAX_TOKENS", 2048) self.deployment_id = self._get("DEPLOYMENT_ID") - self.claude_api_key = self._get('Anthropic_API_KEY') + self.claude_api_key = self._get("Anthropic_API_KEY") self.serpapi_api_key = self._get("SERPAPI_API_KEY") self.serper_api_key = self._get("SERPER_API_KEY") self.google_api_key = self._get("GOOGLE_API_KEY") self.google_cse_id = self._get("GOOGLE_CSE_ID") - self.search_engine = self._get("SEARCH_ENGINE", SearchEngineType.SERPAPI_GOOGLE) - - self.web_browser_engine = WebBrowserEngineType(self._get("WEB_BROWSER_ENGINE", "playwright")) + self.search_engine = SearchEngineType(self._get("SEARCH_ENGINE", SearchEngineType.SERPAPI_GOOGLE)) + self.web_browser_engine = WebBrowserEngineType(self._get("WEB_BROWSER_ENGINE", WebBrowserEngineType.PLAYWRIGHT)) self.playwright_browser_type = self._get("PLAYWRIGHT_BROWSER_TYPE", "chromium") self.selenium_browser_type = self._get("SELENIUM_BROWSER_TYPE", "chrome") - self.long_term_memory = self._get('LONG_TERM_MEMORY', False) + self.long_term_memory = self._get("LONG_TERM_MEMORY", False) if self.long_term_memory: logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 - self.puppeteer_config = self._get("PUPPETEER_CONFIG","") - self.mmdc = self._get("MMDC","mmdc") - self.calc_usage = self._get("CALC_USAGE",True) + self.puppeteer_config = self._get("PUPPETEER_CONFIG", "") + self.mmdc = self._get("MMDC", "mmdc") + self.calc_usage = self._get("CALC_USAGE", True) self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") diff --git a/metagpt/tools/__init__.py b/metagpt/tools/__init__.py index e1f921c05..d98087e4b 100644 --- a/metagpt/tools/__init__.py +++ b/metagpt/tools/__init__.py @@ -7,15 +7,15 @@ """ -from enum import Enum, auto +from enum import Enum class SearchEngineType(Enum): - SERPAPI_GOOGLE = auto() - DIRECT_GOOGLE = auto() - SERPER_GOOGLE = auto() - DUCK_DUCK_GO = auto() - CUSTOM_ENGINE = auto() + SERPAPI_GOOGLE = "serpapi" + SERPER_GOOGLE = "serper" + DIRECT_GOOGLE = "google" + DUCK_DUCK_GO = "ddg" + CUSTOM_ENGINE = "custom" class WebBrowserEngineType(Enum): diff --git a/metagpt/tools/search_engine_ddg.py b/metagpt/tools/search_engine_ddg.py index c054afed1..57bc61b82 100644 --- a/metagpt/tools/search_engine_ddg.py +++ b/metagpt/tools/search_engine_ddg.py @@ -7,11 +7,15 @@ import json from concurrent import futures from typing import Literal, overload -from duckduckgo_search import DDGS -from googleapiclient.errors import HttpError +try: + from duckduckgo_search import DDGS +except ImportError: + raise ImportError( + "To use this module, you should have the `duckduckgo_search` Python package installed. " + "You can install it by running the command: `pip install -e.[search-ddg]`" + ) from metagpt.config import CONFIG -from metagpt.logs import logger class DDGAPIWrapper: @@ -19,6 +23,7 @@ class DDGAPIWrapper: To use this module, you should have the `duckduckgo_search` Python package installed. """ + def __init__( self, *, @@ -77,15 +82,8 @@ class DDGAPIWrapper: query, max_results, ) - try: - search_results = await future - # Extract the search result items from the response + search_results = await future - except HttpError as e: - # Handle errors in the API call - logger.exception(f"fail to search {query} for {e}") - search_results = [] - # Return the list of search result URLs if as_string: return json.dumps(search_results, ensure_ascii=False) @@ -93,11 +91,8 @@ class DDGAPIWrapper: def _search_from_ddgs(self, query: str, max_results: int): return [ - { - "link": i["href"], - "snippet": i["body"], - "title": i["title"] - } for (_, i) in zip(range(max_results), self.ddgs.text(query)) + {"link": i["href"], "snippet": i["body"], "title": i["title"]} + for (_, i) in zip(range(max_results), self.ddgs.text(query)) ] diff --git a/metagpt/tools/search_engine_googleapi.py b/metagpt/tools/search_engine_googleapi.py index c226ca8d2..b9faf2ced 100644 --- a/metagpt/tools/search_engine_googleapi.py +++ b/metagpt/tools/search_engine_googleapi.py @@ -5,30 +5,61 @@ from __future__ import annotations import asyncio import json from concurrent import futures +from typing import Optional from urllib.parse import urlparse import httplib2 -from googleapiclient.discovery import build -from googleapiclient.errors import HttpError +from pydantic import BaseModel, validator from metagpt.config import CONFIG from metagpt.logs import logger +try: + from googleapiclient.discovery import build + from googleapiclient.errors import HttpError +except ImportError: + raise ImportError( + "To use this module, you should have the `google-api-python-client` Python package installed. " + "You can install it by running the command: `pip install -e.[search-google]`" + ) -class GoogleAPIWrapper: - """Wrapper around GoogleAPI. - To use this module, you should have the `google-api-python-client` Python package installed - and set property values for the configurations `GOOGLE_API_KEY` and `GOOGLE_CSE_ID`. See - https://programmablesearchengine.google.com/controlpanel/all. - """ - def __init__( - self, - *, - loop: asyncio.AbstractEventLoop | None = None, - executor: futures.Executor | None = None, - ): - build_kwargs = {"developerKey": CONFIG.google_api_key} +class GoogleAPIWrapper(BaseModel): + google_api_key: Optional[str] = None + google_cse_id: Optional[str] = None + loop: Optional[asyncio.AbstractEventLoop] = None + executor: Optional[futures.Executor] = None + + class Config: + arbitrary_types_allowed = True + + @validator("google_api_key", always=True) + @classmethod + def check_google_api_key(cls, val: str): + val = val or CONFIG.google_api_key + if not val: + raise ValueError( + "To use, make sure you provide the google_api_key when constructing an object. Alternatively, " + "ensure that the environment variable GOOGLE_API_KEY is set with your API key. You can obtain " + "an API key from https://console.cloud.google.com/apis/credentials." + ) + return val + + @validator("google_cse_id", always=True) + @classmethod + def check_google_cse_id(cls, val: str): + val = val or CONFIG.google_cse_id + if not val: + raise ValueError( + "To use, make sure you provide the google_cse_id when constructing an object. Alternatively, " + "ensure that the environment variable GOOGLE_CSE_ID is set with your API key. You can obtain " + "an API key from https://programmablesearchengine.google.com/controlpanel/create." + ) + return val + + @property + def google_api_client(self): + build_kwargs = {"developerKey": self.google_api_key} if CONFIG.global_proxy: parse_result = urlparse(CONFIG.global_proxy) proxy_type = parse_result.scheme @@ -42,10 +73,7 @@ class GoogleAPIWrapper: ), ) service = build("customsearch", "v1", **build_kwargs) - self.google_api_client = service.cse() - self.custom_search_engine_id = CONFIG.google_cse_id - self.loop = loop - self.executor = executor + return service.cse() async def run( self, @@ -69,12 +97,7 @@ class GoogleAPIWrapper: """ loop = self.loop or asyncio.get_event_loop() future = loop.run_in_executor( - self.executor, - self.google_api_client.list( - q=query, - num=max_results, - cx=self.custom_search_engine_id - ).execute + self.executor, self.google_api_client.list(q=query, num=max_results, cx=self.google_cse_id).execute ) try: result = await future @@ -85,13 +108,13 @@ class GoogleAPIWrapper: # Handle errors in the API call logger.exception(f"fail to search {query} for {e}") search_results = [] - + focus = focus or ["snippet", "link", "title"] details = [{i: j for i, j in item_dict.items() if i in focus} for item_dict in search_results] # Return the list of search result URLs if as_string: return safe_google_results(details) - + return details diff --git a/metagpt/tools/search_engine_serpapi.py b/metagpt/tools/search_engine_serpapi.py index 3d2d7cfe4..750184198 100644 --- a/metagpt/tools/search_engine_serpapi.py +++ b/metagpt/tools/search_engine_serpapi.py @@ -8,19 +8,12 @@ from typing import Any, Dict, Optional, Tuple import aiohttp -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, validator -from metagpt.config import Config +from metagpt.config import CONFIG class SerpAPIWrapper(BaseModel): - """Wrapper around SerpAPI. - - To use, you should have the ``google-search-results`` python package installed, - and the environment variable ``SERPAPI_API_KEY`` set with your API key, or pass - `serpapi_api_key` as a named parameter to the constructor. - """ - search_engine: Any #: :meta private: params: dict = Field( default={ @@ -30,14 +23,25 @@ class SerpAPIWrapper(BaseModel): "hl": "en", } ) - config = Config() - serpapi_api_key: Optional[str] = config.serpapi_api_key + serpapi_api_key: Optional[str] = None aiosession: Optional[aiohttp.ClientSession] = None class Config: arbitrary_types_allowed = True - async def run(self, query: str, max_results: int = 8, as_string: bool = True, **kwargs: Any) -> str: + @validator("serpapi_api_key", always=True) + @classmethod + def check_serpapi_api_key(cls, val: str): + val = val or CONFIG.serpapi_api_key + if not val: + raise ValueError( + "To use, make sure you provide the serpapi_api_key when constructing an object. Alternatively, " + "ensure that the environment variable SERPAPI_API_KEY is set with your API key. You can obtain " + "an API key from https://serpapi.com/." + ) + return val + + async def run(self, query, max_results: int = 8, as_string: bool = True, **kwargs: Any) -> str: """Run query through SerpAPI and parse result async.""" return self._process_response(await self.results(query, max_results), as_string=as_string) @@ -48,8 +52,6 @@ class SerpAPIWrapper(BaseModel): params = self.get_params(query) params["source"] = "python" params["num"] = max_results - if self.serpapi_api_key: - params["serp_api_key"] = self.serpapi_api_key params["output"] = "json" url = "https://serpapi.com/search" return url, params @@ -104,7 +106,7 @@ class SerpAPIWrapper(BaseModel): if res.get("organic_results"): toret_l += [get_focused(i) for i in res.get("organic_results")] - return str(toret) + '\n' + str(toret_l) if as_string else toret_l + return str(toret) + "\n" + str(toret_l) if as_string else toret_l if __name__ == "__main__": diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index 2ae2c3b7d..0eec2694b 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -9,33 +9,32 @@ import json from typing import Any, Dict, Optional, Tuple import aiohttp -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, validator -from metagpt.config import Config +from metagpt.config import CONFIG class SerperWrapper(BaseModel): - """Wrapper around SerpAPI. - - To use, you should have the ``google-search-results`` python package installed, - and the environment variable ``SERPAPI_API_KEY`` set with your API key, or pass - `serpapi_api_key` as a named parameter to the constructor. - """ - search_engine: Any #: :meta private: - payload: dict = Field( - default={ - "page": 1, - "num": 10 - } - ) - config = Config() - serper_api_key: Optional[str] = config.serper_api_key + payload: dict = Field(default={"page": 1, "num": 10}) + serper_api_key: Optional[str] = None aiosession: Optional[aiohttp.ClientSession] = None class Config: arbitrary_types_allowed = True + @validator("serper_api_key", always=True) + @classmethod + def check_serper_api_key(cls, val: str): + val = val or CONFIG.serper_api_key + if not val: + raise ValueError( + "To use, make sure you provide the serper_api_key when constructing an object. Alternatively, " + "ensure that the environment variable SERPER_API_KEY is set with your API key. You can obtain " + "an API key from https://serper.dev/." + ) + return val + async def run(self, query: str, max_results: int = 8, as_string: bool = True, **kwargs: Any) -> str: """Run query through Serper and parse result async.""" if isinstance(query, str): @@ -76,18 +75,17 @@ class SerperWrapper(BaseModel): return json.dumps(payloads, sort_keys=True) def get_headers(self) -> Dict[str, str]: - headers = { - 'X-API-KEY': self.serper_api_key, - 'Content-Type': 'application/json' - } + headers = {"X-API-KEY": self.serper_api_key, "Content-Type": "application/json"} return headers @staticmethod def _process_response(res: dict, as_string: bool = False) -> str: """Process response from SerpAPI.""" # logger.debug(res) - focus = ['title', 'snippet', 'link'] - def get_focused(x): return {i: j for i, j in x.items() if i in focus} + focus = ["title", "snippet", "link"] + + def get_focused(x): + return {i: j for i, j in x.items() if i in focus} if "error" in res.keys(): raise ValueError(f"Got error from SerpAPI: {res['error']}") @@ -95,20 +93,11 @@ class SerperWrapper(BaseModel): toret = res["answer_box"]["answer"] elif "answer_box" in res.keys() and "snippet" in res["answer_box"].keys(): toret = res["answer_box"]["snippet"] - elif ( - "answer_box" in res.keys() - and "snippet_highlighted_words" in res["answer_box"].keys() - ): + elif "answer_box" in res.keys() and "snippet_highlighted_words" in res["answer_box"].keys(): toret = res["answer_box"]["snippet_highlighted_words"][0] - elif ( - "sports_results" in res.keys() - and "game_spotlight" in res["sports_results"].keys() - ): + elif "sports_results" in res.keys() and "game_spotlight" in res["sports_results"].keys(): toret = res["sports_results"]["game_spotlight"] - elif ( - "knowledge_graph" in res.keys() - and "description" in res["knowledge_graph"].keys() - ): + elif "knowledge_graph" in res.keys() and "description" in res["knowledge_graph"].keys(): toret = res["knowledge_graph"]["description"] elif "snippet" in res["organic"][0].keys(): toret = res["organic"][0]["snippet"] @@ -121,7 +110,7 @@ class SerperWrapper(BaseModel): if res.get("organic"): toret_l += [get_focused(i) for i in res.get("organic")] - return str(toret) + '\n' + str(toret_l) if as_string else toret_l + return str(toret) + "\n" + str(toret_l) if as_string else toret_l if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index c5440abe0..efc2ea3e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ channels==4.0.0 # chromadb==0.3.22 # Django==4.1.5 # docx==0.2.4 -duckduckgo_search==2.9.4 #faiss==1.5.3 faiss_cpu==1.7.4 fire==0.4.0 diff --git a/setup.py b/setup.py index 2a8edaae7..a88f9de92 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,8 @@ setup( extras_require={ "playwright": ["playwright>=1.26", "beautifulsoup4"], "selenium": ["selenium>4", "webdriver_manager", "beautifulsoup4"], + "search-google": ["google-api-python-client==2.94.0"], + "search-ddg": ["duckduckgo-search==3.8.5"], }, cmdclass={ "install_mermaid": InstallMermaidCLI, From 4da1a7d0ac78dd5641f39bf92c70b289b1d947e6 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Thu, 17 Aug 2023 17:47:03 +0800 Subject: [PATCH 16/22] Fix the 'get_max_completion_tokens' function to prevent reaching the maximum limit. --- metagpt/utils/token_counter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 591bb60f0..a5a65803a 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -96,7 +96,7 @@ def count_string_tokens(string: str, model_name: str) -> int: return len(encoding.encode(string)) -def get_max_completion_tokens(messages: list[dict], model: str, default: int) -> int: +def get_max_completion_tokens(messages: list[dict], model: str, default: int) -> int: """Calculate the maximum number of completion tokens for a given model and list of messages. Args: @@ -108,4 +108,4 @@ def get_max_completion_tokens(messages: list[dict], model: str, default: int) -> """ if model not in TOKEN_MAX: return default - return TOKEN_MAX[model] - count_message_tokens(messages) + return TOKEN_MAX[model] - count_message_tokens(messages) - 1 From d2aaafd23528403c4935db9bfeaaab21b15fe9e0 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Thu, 17 Aug 2023 21:26:11 +0800 Subject: [PATCH 17/22] Suppress the pydantic.ValidationError in SearchAndSummarize --- metagpt/actions/search_and_summarize.py | 27 ++++++++++++++----------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 43dc02838..5e4cdaea0 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -5,6 +5,8 @@ @Author : alexanderwu @File : search_google.py """ +import pydantic + from metagpt.actions import Action from metagpt.config import Config from metagpt.logs import logger @@ -34,7 +36,7 @@ A: MLOps competitors 8. Dataiku """ -SEARCH_AND_SUMMARIZE_SYSTEM_EN_US = SEARCH_AND_SUMMARIZE_SYSTEM.format(LANG='en-us') +SEARCH_AND_SUMMARIZE_SYSTEM_EN_US = SEARCH_AND_SUMMARIZE_SYSTEM.format(LANG="en-us") SEARCH_AND_SUMMARIZE_PROMPT = """ ### Reference Information @@ -102,25 +104,26 @@ class SearchAndSummarize(Action): def __init__(self, name="", context=None, llm=None, engine=None, search_func=None): self.config = Config() self.engine = engine or self.config.search_engine - self.search_engine = SearchEngine(self.engine, run_func=search_func) + + try: + self.search_engine = SearchEngine(self.engine, run_func=search_func) + except pydantic.ValidationError: + self.search_engine = None + self.result = "" super().__init__(name, context, llm) async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str: - no_serpapi = not self.config.serpapi_api_key or 'YOUR_API_KEY' == self.config.serpapi_api_key - no_serper = not self.config.serper_api_key or 'YOUR_API_KEY' == self.config.serper_api_key - no_google = not self.config.google_api_key or 'YOUR_API_KEY' == self.config.google_api_key - - if no_serpapi and no_google and no_serper: - logger.warning('Configure one of SERPAPI_API_KEY, SERPER_API_KEY, GOOGLE_API_KEY to unlock full feature') + if self.search_engine is None: + logger.warning("Configure one of SERPAPI_API_KEY, SERPER_API_KEY, GOOGLE_API_KEY to unlock full feature") return "" - + query = context[-1].content # logger.debug(query) rsp = await self.search_engine.run(query) self.result = rsp if not rsp: - logger.error('empty rsp...') + logger.error("empty rsp...") return "" # logger.info(rsp) @@ -130,8 +133,8 @@ class SearchAndSummarize(Action): # PREFIX = self.prefix, ROLE=self.profile, CONTEXT=rsp, - QUERY_HISTORY='\n'.join([str(i) for i in context[:-1]]), - QUERY=str(context[-1]) + QUERY_HISTORY="\n".join([str(i) for i in context[:-1]]), + QUERY=str(context[-1]), ) result = await self._aask(prompt, system_prompt) logger.debug(prompt) From 7db209f8a0222fa3a28ac29b37ff960dcbce5837 Mon Sep 17 00:00:00 2001 From: Andrei Boca Date: Tue, 22 Aug 2023 17:34:53 +0200 Subject: [PATCH 18/22] fix(startup.py): add custom loop policy for Windows --- startup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/startup.py b/startup.py index f37b5286c..03b2149c4 100644 --- a/startup.py +++ b/startup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import asyncio - +import platform import fire from metagpt.roles import Architect, Engineer, ProductManager, ProjectManager, QaEngineer @@ -33,6 +33,8 @@ def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool :param code_review: Whether to use code review. :return: """ + if platform.system() == "Windows": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.run(startup(idea, investment, n_round, code_review, run_tests)) From 082d566be7ba2e9b9b74debcffe406629ab0d9a0 Mon Sep 17 00:00:00 2001 From: stellaHSR <34952977+stellaHSR@users.noreply.github.com> Date: Fri, 25 Aug 2023 10:22:17 +0800 Subject: [PATCH 19/22] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 33bbd9aba..0528c384e 100644 --- a/README.md +++ b/README.md @@ -229,3 +229,9 @@ ## Contact Information ## Demo https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d + +## Join us +📢 Join Our Discord Channel! +https://discord.gg/4WdszVjv + +Looking forward to seeing you there! 🎉 From db532f138aaffeb9d8d19c4d8af60d1e9127429f Mon Sep 17 00:00:00 2001 From: stellaHSR <34952977+stellaHSR@users.noreply.github.com> Date: Fri, 25 Aug 2023 10:27:24 +0800 Subject: [PATCH 20/22] Update README_CN.md --- docs/README_CN.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/README_CN.md b/docs/README_CN.md index 0ef54b017..e711e2196 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -193,8 +193,11 @@ ## 演示 https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d -## 加入微信讨论群 +## 加入我们 -添加运营小姐姐，拉你入群 +📢 **关于微信群的通知** +经过一段时间的观察和运营，由于微信群运营效率有限，信息淹没、讨论分散，为了提供更好的交流环境和更高效的社区管理，我们诚挚地邀请大家加入我们的Discord社区，与我们一同构建一个更加活跃和有序的交流平台。 -MetaGPT WeChat Discuss Group + +🔗 **Discord邀请链接** ：https://discord.gg/4WdszVjv +感谢大家的支持与理解，期待在Discord上与大家相见！🎉 From f05b79b783177e3714babc834a7a761d621aed86 Mon Sep 17 00:00:00 2001 From: stellaHSR <34952977+stellaHSR@users.noreply.github.com> Date: Fri, 25 Aug 2023 10:27:48 +0800 Subject: [PATCH 21/22] Update README_CN.md --- docs/README_CN.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/README_CN.md b/docs/README_CN.md index e711e2196..9d6c472e7 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -196,8 +196,10 @@ ## 演示 ## 加入我们 📢 **关于微信群的通知** + 经过一段时间的观察和运营，由于微信群运营效率有限，信息淹没、讨论分散，为了提供更好的交流环境和更高效的社区管理，我们诚挚地邀请大家加入我们的Discord社区，与我们一同构建一个更加活跃和有序的交流平台。 🔗 **Discord邀请链接** ：https://discord.gg/4WdszVjv + 感谢大家的支持与理解，期待在Discord上与大家相见！🎉 From 65500204b827fa4cdb05e81408f31d0831fbedc7 Mon Sep 17 00:00:00 2001 From: stellaHSR <34952977+stellaHSR@users.noreply.github.com> Date: Fri, 25 Aug 2023 10:31:11 +0800 Subject: [PATCH 22/22] Update README_CN.md --- docs/README_CN.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/README_CN.md b/docs/README_CN.md index 9d6c472e7..a69123b9b 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -195,11 +195,6 @@ ## 演示 ## 加入我们 -📢 **关于微信群的通知** +📢 加入我们的Discord频道！https://discord.gg/4WdszVjv -经过一段时间的观察和运营，由于微信群运营效率有限，信息淹没、讨论分散，为了提供更好的交流环境和更高效的社区管理，我们诚挚地邀请大家加入我们的Discord社区，与我们一同构建一个更加活跃和有序的交流平台。 - - -🔗 **Discord邀请链接** ：https://discord.gg/4WdszVjv - -感谢大家的支持与理解，期待在Discord上与大家相见！🎉 +期待在那里与您相见！🎉