diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index 06d12e408..46788e306 100644 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -4,4 +4,4 @@ sudo npm install -g @mermaid-js/mermaid-cli # Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using: python --version -python setup.py install \ No newline at end of file +pip install -e. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 537bbc72e..8ab180e28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ COPY . /app/metagpt WORKDIR /app/metagpt RUN mkdir workspace &&\ pip install --no-cache-dir -r requirements.txt &&\ - python setup.py install + pip install -e. # Running with an infinite loop using the tail command CMD ["sh", "-c", "tail -f /dev/null"] diff --git a/README.md b/README.md index 84dafa46b..91a5483e0 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,17 @@ # MetaGPT: The Multi-Agent Framework CN doc EN doc JA doc -Discord Follow -License: MIT +Discord Follow +License: MIT roadmap -roadmap Twitter Follow

+ AgentStore Waitlist Open in Dev Containers Open in GitHub Codespaces + Hugging Face

1. MetaGPT takes a **one line requirement** as input and outputs **user stories / competitive analysis / requirements / data structures / APIs / documents, etc.** @@ -32,6 +33,13 @@ # MetaGPT: The Multi-Agent Framework

Software Company Multi-Role Schematic (Gradually Implementing)

+## MetaGPT's Abilities + + +https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 + + + ## Examples (fully generated by GPT-4) For example, if you type `python startup.py "Design a RecSys like Toutiao"`, you would get many outputs, one of them is data & api design @@ -40,6 +48,9 @@ ## Examples (fully generated by GPT-4) It costs approximately **$0.2** (in GPT-4 API fees) to generate one example with analysis and design, and around **$2.0** for a full project. + + + ## Installation ### Installation Video Guide @@ -49,7 +60,7 @@ ### Installation Video Guide ### Traditional Installation ```bash -# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. +# Step 1: Ensure that NPM is installed on your system. Then install mermaid-js. (If you don't have npm in your computer, please go to the Node.js offical website to install Node.js https://nodejs.org/ and then you will have npm tool in your computer.) npm --version sudo npm install -g @mermaid-js/mermaid-cli @@ -59,7 +70,7 @@ # Step 2: Ensure that Python 3.9+ is installed on your system. You can check thi # Step 3: Clone the repository to your local machine, and install it. git clone https://github.com/geekan/metagpt cd metagpt -python setup.py install +pip install -e. ``` **Note:** @@ -80,7 +91,72 @@ # Step 3: Clone the repository to your local machine, and install it. MMDC: "./node_modules/.bin/mmdc" ``` -- if `python setup.py install` fails with error `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`, try instead running `python setup.py install --user` +- if `pip install -e.` fails with error `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`, try instead running `pip install -e. --user` + +- To convert Mermaid charts to SVG, PNG, and PDF formats. In addition to the Node.js version of Mermaid-CLI, you now have the option to use Python version Playwright, pyppeteer or mermaid.ink for this task. + + - Playwright + - **Install Playwright** + + ```bash + pip install playwright + ``` + + - **Install the Required Browsers** + + to support PDF conversion, please install Chrominum. + + ```bash + playwright install --with-deps chromium + ``` + + - **modify `config.yaml`** + + uncomment MERMAID_ENGINE from config.yaml and change it to `playwright` + + ```yaml + MERMAID_ENGINE: playwright + ``` + + - pyppeteer + - **Install pyppeteer** + + ```bash + pip install pyppeteer + ``` + + - **Use your own Browsers** + + pyppeteer alow you use installed browsers, please set the following envirment + + ```bash + export PUPPETEER_EXECUTABLE_PATH = /path/to/your/chromium or edge or chrome + ``` + + please do not use this command to install browser, it is too old + + ```bash + pyppeteer-install + ``` + + - **modify `config.yaml`** + + uncomment MERMAID_ENGINE from config.yaml and change it to `pyppeteer` + + ```yaml + MERMAID_ENGINE: pyppeteer + ``` + + - mermaid.ink + - **modify `config.yaml`** + + uncomment MERMAID_ENGINE from config.yaml and change it to `ink` + + ```yaml + MERMAID_ENGINE: ink + ``` + + Note: this method does not support pdf export. ### Installation by Docker @@ -214,6 +290,9 @@ ## QuickStart - [MetaGPT quickstart](https://deepwisdom.feishu.cn/wiki/CyY9wdJc4iNqArku3Lncl4v8n2b) +Try it on Huggingface Space +- https://huggingface.co/spaces/deepwisdom/MetaGPT + ## Citation For now, cite the [Arxiv paper](https://arxiv.org/abs/2308.00352): diff --git a/config/config.yaml b/config/config.yaml index 274cdf469..93301fcf2 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -16,11 +16,12 @@ RPM: 10 #Anthropic_API_KEY: "YOUR_API_KEY" #### if AZURE, check https://github.com/openai/openai-cookbook/blob/main/examples/azure/chat.ipynb - +#### You can use ENGINE or DEPLOYMENT mode #OPENAI_API_TYPE: "azure" #OPENAI_API_BASE: "YOUR_AZURE_ENDPOINT" #OPENAI_API_KEY: "YOUR_AZURE_API_KEY" #OPENAI_API_VERSION: "YOUR_AZURE_API_VERSION" +#DEPLOYMENT_NAME: "YOUR_DEPLOYMENT_NAME" #DEPLOYMENT_ID: "YOUR_DEPLOYMENT_ID" #### for Search @@ -75,3 +76,10 @@ SD_T2I_API: "/sdapi/v1/txt2img" ### for Research MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k + +### choose the engine for mermaid conversion, +# default is nodejs, you can change it to playwright,pyppeteer or ink +# MERMAID_ENGINE: nodejs + +### browser path for pyppeteer engine, support Chrome, Chromium,MS Edge +#PYPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable" \ No newline at end of file diff --git a/docs/README_CN.md b/docs/README_CN.md index 2180eb518..1372bf9f4 100644 --- a/docs/README_CN.md +++ b/docs/README_CN.md @@ -9,16 +9,22 @@ # MetaGPT: 多智能体框架

-CN doc -EN doc -JA doc -Discord Follow -License: MIT -roadmap -roadmap +CN doc +EN doc +JA doc +Discord Follow +License: MIT +roadmap Twitter Follow

+

+ AgentStore Waitlist + Open in Dev Containers + Open in GitHub Codespaces + Hugging Face +

+ 1. MetaGPT输入**一句话的老板需求**，输出**用户故事 / 竞品分析 / 需求 / 数据结构 / APIs / 文件等** 2. MetaGPT内部包括**产品经理 / 架构师 / 项目经理 / 工程师**，它提供了一个**软件公司**的全过程与精心调配的SOP 1. `Code = SOP(Team)` 是核心哲学。我们将SOP具象化，并且用于LLM构成的团队 @@ -27,6 +33,11 @@ # MetaGPT: 多智能体框架

软件公司多角色示意图（正在逐步实现）

+## MetaGPT 的能力 + +https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 + + ## 示例（均由 GPT-4 生成） 例如，键入`python startup.py "写个类似今日头条的推荐系统"`并回车，你会获得一系列输出，其一是数据结构与API设计 @@ -50,9 +61,28 @@ # 第 2 步：确保您的系统上安装了 Python 3.9+。您可以使用以下 # 第 3 步：克隆仓库到您的本地机器，并进行安装。 git clone https://github.com/geekan/metagpt cd metagpt -python setup.py install +pip install -e. ``` +**注意：** + +- 如果已经安装了Chrome、Chromium或MS Edge，可以通过将环境变量`PUPPETEER_SKIP_CHROMIUM_DOWNLOAD`设置为`true`来跳过下载Chromium。 + +- 一些人在全局安装此工具时遇到问题。在本地安装是替代解决方案， + + ```bash + npm install @mermaid-js/mermaid-cli + ``` + +- 不要忘记在config.yml中为mmdc配置配置， + + ```yml + PUPPETEER_CONFIG: "./config/puppeteer-config.json" + MMDC: "./node_modules/.bin/mmdc" + ``` + +- 如果`pip install -e.`失败并显示错误`[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'`，请尝试使用`pip install -e. --user`运行。 + ### Docker安装 ```bash @@ -106,16 +136,16 @@ # 复制配置文件并进行必要的修改 cp config/config.yaml config/key.yaml ``` -| 变量名 | config/key.yaml | env | -|--------------------------------------------|-------------------------------------------|--------------------------------| -| OPENAI_API_KEY # 用您自己的密钥替换 | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | -| OPENAI_API_BASE # 可选 | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | +| 变量名 | config/key.yaml | env | +| ----------------------------------- | ----------------------------------------- | ----------------------------------------------- | +| OPENAI_API_KEY # 用您自己的密钥替换 | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | +| OPENAI_API_BASE # 可选 | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | ## 示例：启动一个创业公司 ```shell python startup.py "写一个命令行贪吃蛇" -# 开启code review模式会会花费更多的money, 但是会提升代码质量和成功率 +# 开启code review模式会花费更多的金钱, 但是会提升代码质量和成功率 python startup.py "写一个命令行贪吃蛇" --code_review True ``` @@ -123,7 +153,6 @@ # 开启code review模式会会花费更多的money, 但是会提升代码质量 ### 平台或工具的倾向性 可以在阐述需求时说明想要使用的平台或工具。 例如： - ```shell python startup.py "写一个基于pygame的命令行贪吃蛇" ``` @@ -180,6 +209,10 @@ ## 快速体验 - [MetaGPT快速体验](https://deepwisdom.feishu.cn/wiki/Q8ycw6J9tiNXdHk66MRcIN8Pnlg) +可直接在Huggingface Space体验 + +- https://huggingface.co/spaces/deepwisdom/MetaGPT + ## 联系信息 如果您对这个项目有任何问题或反馈，欢迎联系我们。我们非常欢迎您的建议！ diff --git a/docs/README_JA.md b/docs/README_JA.md index 4be862b6f..8d6c2fe84 100644 --- a/docs/README_JA.md +++ b/docs/README_JA.md @@ -9,16 +9,22 @@ # MetaGPT: マルチエージェントフレームワーク

-CN doc -EN doc -JA doc -Discord Follow -License: MIT +CN doc +EN doc +JA doc +Discord Follow +License: MIT roadmap -roadmap Twitter Follow

+

+ AgentStore Waitlist + Open in Dev Containers + Open in GitHub Codespaces + Hugging Face +

+ 1. MetaGPT は、**1 行の要件** を入力とし、**ユーザーストーリー / 競合分析 / 要件 / データ構造 / API / 文書など** を出力します。 2. MetaGPT には、**プロダクト マネージャー、アーキテクト、プロジェクト マネージャー、エンジニア** が含まれています。MetaGPT は、**ソフトウェア会社のプロセス全体を、慎重に調整された SOP とともに提供します。** 1. `Code = SOP(Team)` が基本理念です。私たちは SOP を具体化し、LLM で構成されるチームに適用します。 @@ -27,6 +33,11 @@ # MetaGPT: マルチエージェントフレームワーク

ソフトウェア会社のマルチロール図式（順次導入）

+## MetaGPTの能力 + +https://github.com/geekan/MetaGPT/assets/34952977/34345016-5d13-489d-b9f9-b82ace413419 + + ## 例（GPT-4 で完全生成） 例えば、`python startup.py "Toutiao のような RecSys をデザインする"`と入力すると、多くの出力が得られます @@ -37,6 +48,10 @@ ## 例（GPT-4 で完全生成） ## インストール +### インストールビデオガイド + +- [Matthew Berman: How To Install MetaGPT - Build A Startup With One Prompt!!](https://youtu.be/uT75J_KG_aY) + ### 伝統的なインストール ```bash @@ -50,7 +65,7 @@ # ステップ 2: Python 3.9+ がシステムにインストールされてい # ステップ 3: リポジトリをローカルマシンにクローンし、インストールする。 git clone https://github.com/geekan/metagpt cd metagpt -python setup.py install +pip install -e. ``` **注:** @@ -60,18 +75,18 @@ # ステップ 3: リポジトリをローカルマシンにクローンし、 - このツールをグローバルにインストールする[問題を抱えている](https://github.com/mermaidjs/mermaid.cli/issues/15)人もいます。ローカルにインストールするのが代替の解決策です、 - ```bash - npm install @mermaid-js/mermaid-cli - ``` + ```bash + npm install @mermaid-js/mermaid-cli + ``` - config.yml に mmdc のコンフィギュレーションを記述するのを忘れないこと - ```yml - PUPPETEER_CONFIG: "./config/puppeteer-config.json" - MMDC: "./node_modules/.bin/mmdc" - ``` + ```yml + PUPPETEER_CONFIG: "./config/puppeteer-config.json" + MMDC: "./node_modules/.bin/mmdc" + ``` -- もし `python setup.py install` がエラー `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'` で失敗したら、代わりに `python setup.py install --user` を実行してみてください +- もし `pip install -e.` がエラー `[Errno 13] Permission denied: '/usr/local/lib/python3.11/dist-packages/test-easy-install-13129.write-test'` で失敗したら、代わりに `pip install -e. --user` を実行してみてください ### Docker によるインストール @@ -126,26 +141,32 @@ # 設定ファイルをコピーし、必要な修正を加える。 cp config/config.yaml config/key.yaml ``` -| 変数名 | config/key.yaml | env | -| ------------------------------------------ | ----------------------------------------- | ----------------------------------------------- | -| OPENAI_API_KEY # 自分のキーに置き換える | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | -| OPENAI_API_BASE # オプション | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | +| 変数名 | config/key.yaml | env | +| --------------------------------------- | ----------------------------------------- | ----------------------------------------------- | +| OPENAI_API_KEY # 自分のキーに置き換える | OPENAI_API_KEY: "sk-..." | export OPENAI_API_KEY="sk-..." | +| OPENAI_API_BASE # オプション | OPENAI_API_BASE: "https:///v1" | export OPENAI_API_BASE="https:///v1" | ## チュートリアル: スタートアップの開始 ```shell +# スクリプトの実行 python startup.py "Write a cli snake game" -# コードレビューを利用すれば、コストはかかるが、より良いコード品質を選ぶことができます。 +# プロジェクトの実施にエンジニアを雇わないこと +python startup.py "Write a cli snake game" --implement False +# エンジニアを雇い、コードレビューを行う python startup.py "Write a cli snake game" --code_review True ``` スクリプトを実行すると、`workspace/` ディレクトリに新しいプロジェクトが見つかります。 + ### プラットフォームまたはツールの設定 要件を述べるときに、どのプラットフォームまたはツールを使用するかを指定できます。 + ```shell python startup.py "pygame をベースとした cli ヘビゲームを書く" ``` + ### 使用方法 ``` @@ -194,13 +215,18 @@ ### コードウォークスルー `examples` でシングル・ロール（ナレッジ・ベース付き）と LLM のみの例を詳しく見ることができます。 ## クイックスタート + ローカル環境のインストールや設定は、ユーザーによっては難しいものです。以下のチュートリアルで MetaGPT の魅力をすぐに体験できます。 - [MetaGPT クイックスタート](https://deepwisdom.feishu.cn/wiki/CyY9wdJc4iNqArku3Lncl4v8n2b) +Hugging Face Space で試す +- https://huggingface.co/spaces/deepwisdom/MetaGPT + ## 引用 現時点では、[Arxiv 論文](https://arxiv.org/abs/2308.00352)を引用してください: + ```bibtex @misc{hong2023metagpt, title={MetaGPT: Meta Programming for Multi-Agent Collaborative Framework}, @@ -224,3 +250,10 @@ ## お問い合わせ先 ## デモ https://github.com/geekan/MetaGPT/assets/2707039/5e8c1062-8c35-440f-bb20-2b0320f8d27d + +## 参加する + +📢 Discord チャンネルに参加してください！ +https://discord.gg/ZRHeExS6xv + +お会いできることを楽しみにしています！ 🎉 diff --git a/docs/resources/20230811-214014.jpg b/docs/resources/20230811-214014.jpg deleted file mode 100644 index 2006f2646..000000000 Binary files a/docs/resources/20230811-214014.jpg and /dev/null differ diff --git a/docs/resources/MetaGPT-WeChat-Personal.jpeg b/docs/resources/MetaGPT-WeChat-Personal.jpeg deleted file mode 100644 index f6b48577d..000000000 Binary files a/docs/resources/MetaGPT-WeChat-Personal.jpeg and /dev/null differ diff --git a/docs/resources/MetaGPT-WorkWeChatGroup-6.jpg b/docs/resources/MetaGPT-WorkWeChatGroup-6.jpg deleted file mode 100644 index 77a4668f7..000000000 Binary files a/docs/resources/MetaGPT-WorkWeChatGroup-6.jpg and /dev/null differ diff --git a/examples/write_tutorial.py b/examples/write_tutorial.py new file mode 100644 index 000000000..167f3eb7c --- /dev/null +++ b/examples/write_tutorial.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# _*_ coding: utf-8 _*_ +""" +@Time : 2023/9/4 21:40:57 +@Author : Stitch-z +@File : tutorial_assistant.py +""" +import asyncio + +from metagpt.roles.tutorial_assistant import TutorialAssistant + + +async def main(): + topic = "Write a tutorial about MySQL" + role = TutorialAssistant(language="Chinese") + await role.run(topic) + + +if __name__ == '__main__': + asyncio.run(main()) \ No newline at end of file diff --git a/metagpt/__init__.py b/metagpt/__init__.py index b9c530d24..71ddd1aff 100644 --- a/metagpt/__init__.py +++ b/metagpt/__init__.py @@ -1,5 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 2023/4/24 22:26 # @Author : alexanderwu # @File : __init__.py + +from metagpt import _compat as _ # noqa: F401 diff --git a/metagpt/_compat.py b/metagpt/_compat.py new file mode 100644 index 000000000..e94a4d095 --- /dev/null +++ b/metagpt/_compat.py @@ -0,0 +1,15 @@ +import platform +import sys +import warnings + +if sys.implementation.name == "cpython" and platform.system() == "Windows" and sys.version_info[:2] == (3, 9): + # https://github.com/python/cpython/pull/92842 + + from asyncio.proactor_events import _ProactorBasePipeTransport + + def pacth_del(self, _warn=warnings.warn): + if self._sock is not None: + _warn(f"unclosed transport {self!r}", ResourceWarning, source=self) + self._sock.close() + + _ProactorBasePipeTransport.__del__ = pacth_del diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index cde366533..89e2ad93d 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -92,30 +92,30 @@ class WriteDesign(Action): pass # Folder does not exist, but we don't care workspace.mkdir(parents=True, exist_ok=True) - def _save_prd(self, docs_path, resources_path, context): + async def _save_prd(self, docs_path, resources_path, context): prd_file = docs_path / "prd.md" if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"] - mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") + await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") if context[-1].instruct_content: logger.info(f"Saving PRD to {prd_file}") prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict())) - def _save_system_design(self, docs_path, resources_path, system_design): + async def _save_system_design(self, docs_path, resources_path, system_design): data_api_design = system_design.instruct_content.dict()[ "Data structures and interface definitions" ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) seq_flow = system_design.instruct_content.dict()[ "Program call flow" ] # CodeParser.parse_code(block="Program call flow", text=content) - mermaid_to_file(data_api_design, resources_path / "data_api_design") - mermaid_to_file(seq_flow, resources_path / "seq_flow") + await mermaid_to_file(data_api_design, resources_path / "data_api_design") + await mermaid_to_file(seq_flow, resources_path / "seq_flow") system_design_file = docs_path / "system_design.md" logger.info(f"Saving System Designs to {system_design_file}") system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict()))) - def _save(self, context, system_design): + async def _save(self, context, system_design): if isinstance(system_design, ActionOutput): ws_name = system_design.instruct_content.dict()["Python package name"] else: @@ -126,12 +126,12 @@ class WriteDesign(Action): resources_path = workspace / "resources" docs_path.mkdir(parents=True, exist_ok=True) resources_path.mkdir(parents=True, exist_ok=True) - self._save_prd(docs_path, resources_path, context) - self._save_system_design(docs_path, resources_path, system_design) + await self._save_prd(docs_path, resources_path, context) + await self._save_system_design(docs_path, resources_path, system_design) async def run(self, context): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) # system_design = await self._aask(prompt) system_design = await self._aask_json_v1(prompt, "system_design", OUTPUT_MAPPING) - self._save(context, system_design) + await self._save(context, system_design) return system_design diff --git a/metagpt/actions/write_tutorial.py b/metagpt/actions/write_tutorial.py new file mode 100644 index 000000000..b23fc2ad4 --- /dev/null +++ b/metagpt/actions/write_tutorial.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# _*_ coding: utf-8 _*_ +""" +@Time : 2023/9/4 15:40:40 +@Author : Stitch-z +@File : tutorial_assistant.py +@Describe : Actions of the tutorial assistant, including writing directories and document content. +""" +import json +from typing import Dict + +from metagpt.actions import Action +from metagpt.logs import logger +from metagpt.prompts.tutorial_assistant import DIRECTORY_PROMPT, CONTENT_PROMPT + + +class WriteDirectory(Action): + """Action class for writing tutorial directories. + + Args: + name: The name of the action. + language: The language to output, default is "Chinese". + """ + + def __init__(self, name: str = "", language: str = "Chinese", *args, **kwargs): + super().__init__(name, *args, **kwargs) + self.language = language + + @staticmethod + async def _handle_resp(resp: str) -> Dict: + """Process string results and convert them to JSON format. + + Args: + resp: The directory results returned by gpt. + + Returns: + The parsed dictionary, such as {"title": "xxx", "directory": [{"dir 1": ["sub dir 1", "sub dir 2"]}]}. + + Raises: + Exception: If no matching dictionary section is found. + json.JSONDecodeError: If the dictionary part cannot be parsed as JSON. + """ + start = resp.find('{') + end = resp.rfind('}') + if start != -1 and end != -1 and end > start: + directory_str = resp[start:end + 1] + logger.info(f"Successfully parsed json: {str(directory_str)}") + try: + return json.loads(directory_str) + except json.JSONDecodeError as e: + logger.error(f"Json parsing error: {e}") + raise e + else: + raise Exception("No matching dictionary section found.") + + async def run(self, topic: str, *args, **kwargs) -> Dict: + """Execute the action to generate a tutorial directory according to the topic. + + Args: + topic: The tutorial topic. + + Returns: + the tutorial directory information, including {"title": "xxx", "directory": [{"dir 1": ["sub dir 1", "sub dir 2"]}]}. + """ + prompt = DIRECTORY_PROMPT.format(topic=topic, language=self.language) + resp = await self._aask(prompt=prompt) + return await self._handle_resp(resp) + + +class WriteContent(Action): + """Action class for writing tutorial content. + + Args: + name: The name of the action. + directory: The content to write. + language: The language to output, default is "Chinese". + """ + + def __init__(self, name: str = "", directory: str = "", language: str = "Chinese", *args, **kwargs): + super().__init__(name, *args, **kwargs) + self.language = language + self.directory = directory + + async def run(self, topic: str, *args, **kwargs) -> str: + """Execute the action to write document content according to the directory and topic. + + Args: + topic: The tutorial topic. + + Returns: + The written tutorial content. + """ + prompt = CONTENT_PROMPT.format(topic=topic, language=self.language, directory=self.directory) + return await self._aask(prompt=prompt) + diff --git a/metagpt/config.py b/metagpt/config.py index 76c6563cb..b4e0fe7fa 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -59,6 +59,7 @@ class Config(metaclass=Singleton): self.openai_api_rpm = self._get("RPM", 3) self.openai_api_model = self._get("OPENAI_API_MODEL", "gpt-4") self.max_tokens_rsp = self._get("MAX_TOKENS", 2048) + self.deployment_name = self._get('DEPLOYMENT_NAME') self.deployment_id = self._get("DEPLOYMENT_ID") self.claude_api_key = self._get("Anthropic_API_KEY") @@ -82,6 +83,8 @@ class Config(metaclass=Singleton): self.calc_usage = self._get("CALC_USAGE", True) self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") + self.mermaid_engine = self._get("MERMAID_ENGINE", 'nodejs') + self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", '') def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" diff --git a/metagpt/const.py b/metagpt/const.py index 16f652186..35b4c9fa7 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -33,5 +33,6 @@ API_QUESTIONS_PATH = UT_PATH / "files/question/" YAPI_URL = "http://yapi.deepwisdomai.com/" TMP = PROJECT_ROOT / 'tmp' RESEARCH_PATH = DATA_PATH / "research" +TUTORIAL_PATH = DATA_PATH / "tutorial_docx" MEM_TTL = 24 * 30 * 3600 diff --git a/metagpt/prompts/tutorial_assistant.py b/metagpt/prompts/tutorial_assistant.py new file mode 100644 index 000000000..d690aad83 --- /dev/null +++ b/metagpt/prompts/tutorial_assistant.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# _*_ coding: utf-8 _*_ +""" +@Time : 2023/9/4 15:40:40 +@Author : Stitch-z +@File : tutorial_assistant.py +@Describe : Tutorial Assistant's prompt templates. +""" + +COMMON_PROMPT = """ +You are now a seasoned technical professional in the field of the internet. +We need you to write a technical tutorial with the topic "{topic}". +""" + +DIRECTORY_PROMPT = COMMON_PROMPT + """ +Please provide the specific table of contents for this tutorial, strictly following the following requirements: +1. The output must be strictly in the specified language, {language}. +2. Answer strictly in the dictionary format like {{"title": "xxx", "directory": [{{"dir 1": ["sub dir 1", "sub dir 2"]}}, {{"dir 2": ["sub dir 3", "sub dir 4"]}}]}}. +3. The directory should be as specific and sufficient as possible, with a primary and secondary directory.The secondary directory is in the array. +4. Do not have extra spaces or line breaks. +5. Each directory title has practical significance. +""" + +CONTENT_PROMPT = COMMON_PROMPT + """ +Now I will give you the module directory titles for the topic. +Please output the detailed principle content of this title in detail. +If there are code examples, please provide them according to standard code specifications. +Without a code example, it is not necessary. + +The module directory titles for the topic is as follows: +{directory} + +Strictly limit output according to the following requirements: +1. Follow the Markdown syntax format for layout. +2. If there are code examples, they must follow standard syntax specifications, have document annotations, and be displayed in code blocks. +3. The output must be strictly in the specified language, {language}. +4. Do not have redundant output, including concluding remarks. +5. Strict requirement not to output the topic "{topic}". +""" \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index ff8943d3b..7e865f288 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -6,11 +6,17 @@ """ import asyncio import time -from typing import NamedTuple +from typing import NamedTuple, Union import openai from openai.error import APIConnectionError -from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type +from tenacity import ( + after_log, + retry, + retry_if_exception_type, + stop_after_attempt, + wait_fixed, +) from metagpt.config import CONFIG from metagpt.logs import logger @@ -48,12 +54,14 @@ class RateLimiter: self.last_call_time = time.time() + class Costs(NamedTuple): total_prompt_tokens: int total_completion_tokens: int total_cost: float total_budget: float + class CostManager(metaclass=Singleton): """计算使用接口的开销""" @@ -74,7 +82,9 @@ class CostManager(metaclass=Singleton): """ self.total_prompt_tokens += prompt_tokens self.total_completion_tokens += completion_tokens - cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"]) / 1000 + cost = ( + prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"] + ) / 1000 self.total_cost += cost logger.info( f"Total running cost: ${self.total_cost:.3f} | Max budget: ${CONFIG.max_budget:.3f} | " @@ -100,6 +110,7 @@ class CostManager(metaclass=Singleton): """ return self.total_completion_tokens + def get_total_cost(self): """ Get the total cost of API calls. @@ -109,25 +120,20 @@ def get_total_cost(self): """ return self.total_cost + def get_costs(self) -> Costs: """Get all costs""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) -def log_and_reraise(retry_state): - logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") - logger.warning(""" -Recommend going to https://deepwisdom.feishu.cn/wiki/MsGnwQBjiif9c3koSJNcYaoSnu4#part-XdatdVlhEojeAfxaaEZcMV3ZniQ -See FAQ 5.8 -""") - raise retry_state.outcome.exception() - def log_and_reraise(retry_state): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") - logger.warning(""" + logger.warning( + """ Recommend going to https://deepwisdom.feishu.cn/wiki/MsGnwQBjiif9c3koSJNcYaoSnu4#part-XdatdVlhEojeAfxaaEZcMV3ZniQ See FAQ 5.8 -""") +""" + ) raise retry_state.outcome.exception() @@ -162,10 +168,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # iterate through the stream of events async for chunk in response: collected_chunks.append(chunk) # save the event response - chunk_message = chunk["choices"][0]["delta"] # extract the message - collected_messages.append(chunk_message) # save the message - if "content" in chunk_message: - print(chunk_message["content"], end="") + choices = chunk["choices"] + if len(choices) > 0: + chunk_message = chunk["choices"][0].get("delta", {}) # extract the message + collected_messages.append(chunk_message) # save the message + if "content" in chunk_message: + print(chunk_message["content"], end="") print() full_reply_content = "".join([m.get("content", "") for m in collected_messages]) @@ -174,25 +182,27 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return full_reply_content def _cons_kwargs(self, messages: list[dict]) -> dict: + kwargs = { + "messages": messages, + "max_tokens": self.get_max_tokens(messages), + "n": 1, + "stop": None, + "temperature": 0.3, + "timeout": 3, + } if CONFIG.openai_api_type == "azure": - kwargs = { - "deployment_id": CONFIG.deployment_id, - "messages": messages, - "max_tokens": self.get_max_tokens(messages), - "n": 1, - "stop": None, - "temperature": 0.3, - } + if CONFIG.deployment_name and CONFIG.deployment_id: + raise ValueError("You can only use one of the `deployment_id` or `deployment_name` model") + elif not CONFIG.deployment_name and not CONFIG.deployment_id: + raise ValueError("You must specify `DEPLOYMENT_NAME` or `DEPLOYMENT_ID` parameter") + kwargs_mode = ( + {"engine": CONFIG.deployment_name} + if CONFIG.deployment_name + else {"deployment_id": CONFIG.deployment_id} + ) else: - kwargs = { - "model": self.model, - "messages": messages, - "max_tokens": self.get_max_tokens(messages), - "n": 1, - "stop": None, - "temperature": 0.3, - } - kwargs["timeout"] = 3 + kwargs_mode = {"model": self.model} + kwargs.update(kwargs_mode) return kwargs async def _achat_completion(self, messages: list[dict]) -> dict: @@ -218,7 +228,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @retry( stop=stop_after_attempt(3), wait=wait_fixed(1), - after=after_log(logger, logger.level('WARNING').name), + after=after_log(logger, logger.level("WARNING").name), retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) @@ -235,8 +245,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): try: prompt_tokens = count_message_tokens(messages, self.model) completion_tokens = count_string_tokens(rsp, self.model) - usage['prompt_tokens'] = prompt_tokens - usage['completion_tokens'] = completion_tokens + usage["prompt_tokens"] = prompt_tokens + usage["completion_tokens"] = completion_tokens return usage except Exception as e: logger.error("usage calculation failed!", e) @@ -272,8 +282,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def _update_costs(self, usage: dict): if CONFIG.calc_usage: try: - prompt_tokens = int(usage['prompt_tokens']) - completion_tokens = int(usage['completion_tokens']) + prompt_tokens = int(usage["prompt_tokens"]) + completion_tokens = int(usage["completion_tokens"]) self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) except Exception as e: logger.error("updating costs failed!", e) @@ -285,3 +295,31 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if not self.auto_max_tokens: return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) + + def moderation(self, content: Union[str, list[str]]): + try: + if not content: + logger.error("content cannot be empty!") + else: + rsp = self._moderation(content=content) + return rsp + except Exception as e: + logger.error(f"moderating failed:{e}") + + def _moderation(self, content: Union[str, list[str]]): + rsp = self.llm.Moderation.create(input=content) + return rsp + + async def amoderation(self, content: Union[str, list[str]]): + try: + if not content: + logger.error("content cannot be empty!") + else: + rsp = await self._amoderation(content=content) + return rsp + except Exception as e: + logger.error(f"moderating failed:{e}") + + async def _amoderation(self, content: Union[str, list[str]]): + rsp = await self.llm.Moderation.acreate(input=content) + return rsp diff --git a/metagpt/roles/prompt.py b/metagpt/roles/prompt.py index d13551203..c22e0226b 100644 --- a/metagpt/roles/prompt.py +++ b/metagpt/roles/prompt.py @@ -32,7 +32,7 @@ class PromptString(Enum): RECENT_ACTIVITY = "Based on the following memory, produce a brief summary of what {full_name} has been up to recently. Do not invent details not explicitly stated in the memory. For any conversation, be sure to mention whether the conversation has concluded or is still ongoing.\n\nMemory: {memory_descriptions}" - MAKE_PLANS = 'You are a plan-generating AI. Your job is to assist the character in formulating new plans based on new information. Given the character's information (profile, objectives, recent activities, current plans, and location context) and their current thought process, produce a new set of plans for them. The final plan should comprise at least {time_window} of activities and no more than 5 individual plans. List the plans in the order they should be executed, with each plan detailing its description, location, start time, stop criteria, and maximum duration.\n\nSample plan: \'{{"index": 1, "description": "Cook dinner", "location_id": "0a3bc22b-36aa-48ab-adb0-18616004caed","start_time": "2022-12-12T20:00:00+00:00","max_duration_hrs": 1.5, "stop_condition": "Dinner is fully prepared"}}\'\n\nFor each plan, choose the most appropriate location name from this list: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize completing any unfinished conversations.\n\nLet's begin!\n\nName: {full_name}\nProfile: {private_bio}\nObjectives: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activities: {recent_activity}\nThought Process: {thought_process}\nIt's essential to encourage the character to collaborate with other characters in their plans.\n\n' + MAKE_PLANS = "You are a plan-generating AI. Your job is to assist the character in formulating new plans based on new information. Given the character's information (profile, objectives, recent activities, current plans, and location context) and their current thought process, produce a new set of plans for them. The final plan should comprise at least {time_window} of activities and no more than 5 individual plans. List the plans in the order they should be executed, with each plan detailing its description, location, start time, stop criteria, and maximum duration.\n\nSample plan: {{\"index\": 1, \"description\": \"Cook dinner\", \"location_id\": \"0a3bc22b-36aa-48ab-adb0-18616004caed\",\"start_time\": \"2022-12-12T20:00:00+00:00\",\"max_duration_hrs\": 1.5, \"stop_condition\": \"Dinner is fully prepared\"}}\'\n\nFor each plan, choose the most appropriate location name from this list: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize completing any unfinished conversations.\n\nLet's begin!\n\nName: {full_name}\nProfile: {private_bio}\nObjectives: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activities: {recent_activity}\nThought Process: {thought_process}\nIt's essential to encourage the character to collaborate with other characters in their plans.\n\n" EXECUTE_PLAN = "You are a role-playing AI, playing the role of {your_name}, in front of a live audience. Every word you say can be observed by the audience, so make sure you speak often and make it interesting. You cannot interact directly with the audience.\n\nBased on the following context and tools, proceed as if you are {your_name}. Your primary task is to complete the task given below, however, if you are currently in a conversation with another character, you should always complete the conversation first and then proceed with the task. Do not start working while you are engaged in an unfinished conversation. Use your best judgment to determine whether a conversation involves you and whether it is unfinished. You don't need to respond to every message you receive.\n\nThis is a personal profile of your character:\n{your_private_bio}\n\nThis is a description of your location and other nearby characters you can converse with:\n\n{location_context}\n\nThese are some relevant memories:\n```\n{relevant_memories}\n```\n\nThis is some relevant conversation history:\n```\n{conversation_history}\n```\n\nThese are the tools you can use:\n{tools}\n\nYour response should follow the format below:\n\nTask: The task you must complete\nThoughts: What you should think about doing\nAction: The action to take, must be one of these words: [{tool_names}]\nAction Input: Input for the action\nObservation: The result of the action\n... (This Thoughts/Action/Action Input/Observation can be repeated N times)\nThoughts: 'I have completed the task'\nFinal Response: The final response to the task\n\nIf you are not ready with a final response, then you must take an action.\n\nIf you are certain that you cannot complete the task with the tools provided, return 'Final Response: Need Help', however, if you are in conversation with another character, responses like 'I don't know' are valid. During conversations, you should never break character or admit you are an AI.\nIf the task is completed and no specific response is required, return 'Final Response: Done'\nLet's begin!\n\nTask: {input}\n\n{agent_scratchpad}" diff --git a/metagpt/roles/tutorial_assistant.py b/metagpt/roles/tutorial_assistant.py new file mode 100644 index 000000000..9a7df4f4d --- /dev/null +++ b/metagpt/roles/tutorial_assistant.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +# _*_ coding: utf-8 _*_ +""" +@Time : 2023/9/4 15:40:40 +@Author : Stitch-z +@File : tutorial_assistant.py +""" + +from datetime import datetime +from typing import Dict + +from metagpt.actions.write_tutorial import WriteDirectory, WriteContent +from metagpt.const import TUTORIAL_PATH +from metagpt.logs import logger +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.utils.file import File + + +class TutorialAssistant(Role): + """Tutorial assistant, input one sentence to generate a tutorial document in markup format. + + Args: + name: The name of the role. + profile: The role profile description. + goal: The goal of the role. + constraints: Constraints or requirements for the role. + language: The language in which the tutorial documents will be generated. + """ + + def __init__( + self, + name: str = "Stitch", + profile: str = "Tutorial Assistant", + goal: str = "Generate tutorial documents", + constraints: str = "Strictly follow Markdown's syntax, with neat and standardized layout", + language: str = "Chinese", + ): + super().__init__(name, profile, goal, constraints) + self._init_actions([WriteDirectory(language=language)]) + self.topic = "" + self.main_title = "" + self.total_content = "" + self.language = language + + async def _think(self) -> None: + """Determine the next action to be taken by the role.""" + if self._rc.todo is None: + self._set_state(0) + return + + if self._rc.state + 1 < len(self._states): + self._set_state(self._rc.state + 1) + else: + self._rc.todo = None + + async def _handle_directory(self, titles: Dict) -> Message: + """Handle the directories for the tutorial document. + + Args: + titles: A dictionary containing the titles and directory structure, + such as {"title": "xxx", "directory": [{"dir 1": ["sub dir 1", "sub dir 2"]}]} + + Returns: + A message containing information about the directory. + """ + self.main_title = titles.get("title") + directory = f"{self.main_title}\n" + self.total_content += f"# {self.main_title}" + actions = list() + for first_dir in titles.get("directory"): + actions.append(WriteContent(language=self.language, directory=first_dir)) + key = list(first_dir.keys())[0] + directory += f"- {key}\n" + for second_dir in first_dir[key]: + directory += f" - {second_dir}\n" + self._init_actions(actions) + self._rc.todo = None + return Message(content=directory) + + async def _act(self) -> Message: + """Perform an action as determined by the role. + + Returns: + A message containing the result of the action. + """ + todo = self._rc.todo + if type(todo) is WriteDirectory: + msg = self._rc.memory.get(k=1)[0] + self.topic = msg.content + resp = await todo.run(topic=self.topic) + logger.info(resp) + return await self._handle_directory(resp) + resp = await todo.run(topic=self.topic) + logger.info(resp) + if self.total_content != "": + self.total_content += "\n\n\n" + self.total_content += resp + return Message(content=resp, role=self.profile) + + async def _react(self) -> Message: + """Execute the assistant's think and actions. + + Returns: + A message containing the final result of the assistant's actions. + """ + while True: + await self._think() + if self._rc.todo is None: + break + msg = await self._act() + root_path = TUTORIAL_PATH / datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + await File.write(root_path, f"{self.main_title}.md", self.total_content.encode('utf-8')) + return msg diff --git a/metagpt/tools/ut_writer.py b/metagpt/tools/ut_writer.py index 263a0269e..43ca72150 100644 --- a/metagpt/tools/ut_writer.py +++ b/metagpt/tools/ut_writer.py @@ -60,6 +60,7 @@ def test_node_tags(project_key, nodes, operations, expected_msg): # 3. If comments are needed, use Chinese. # If you understand, please wait for me to give the interface definition and just answer "Understood" to save tokens. +''' ACT_PROMPT_PREFIX = '''Refer to the test types: such as missing request parameters, field boundary verification, incorrect field type. Please output 10 test cases within one `@pytest.mark.parametrize` scope. @@ -94,7 +95,8 @@ Name Type Required Default Value Remarks code integer Yes message string Yes data object Yes - +``` +''' class UTGenerator: diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 7f090cf63..5f94de066 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -9,6 +9,7 @@ import ast import contextlib import inspect import os +import platform import re from typing import List, Tuple @@ -20,7 +21,10 @@ def check_cmd_exists(command) -> int: :param command: 待检查的命令 :return: 如果命令存在，返回0，如果不存在，返回非0 """ - check_command = 'command -v ' + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }' + if platform.system().lower() == 'windows': + check_command = 'where ' + command + else: + check_command = 'command -v ' + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }' result = os.system(check_command) return result @@ -187,7 +191,8 @@ class CodeParser: else: logger.error(f"{pattern} not match following text:") logger.error(text) - raise Exception + # raise Exception + return "" return code @classmethod diff --git a/metagpt/utils/file.py b/metagpt/utils/file.py new file mode 100644 index 000000000..5aca2a0e5 --- /dev/null +++ b/metagpt/utils/file.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# _*_ coding: utf-8 _*_ +""" +@Time : 2023/9/4 15:40:40 +@Author : Stitch-z +@File : file.py +@Describe : General file operations. +""" +import aiofiles +from pathlib import Path + +from metagpt.logs import logger + + +class File: + """A general util for file operations.""" + + @classmethod + async def write(cls, root_path: Path, filename: str, content: bytes) -> Path: + """Write the file content to the local specified path. + + Args: + root_path: The root path of file, such as "/data". + filename: The name of file, such as "test.txt". + content: The binary content of file. + + Returns: + The full filename of file, such as "/data/test.txt". + + Raises: + Exception: If an unexpected error occurs during the file writing process. + """ + try: + root_path.mkdir(parents=True, exist_ok=True) + full_path = root_path / filename + async with aiofiles.open(full_path, mode="wb") as writer: + await writer.write(content) + logger.info(f"Successfully write file: {full_path}") + return full_path + except Exception as e: + logger.error(f"Error writing file: {e}") + raise e \ No newline at end of file diff --git a/metagpt/utils/index.html b/metagpt/utils/index.html new file mode 100644 index 000000000..d750a1b6a --- /dev/null +++ b/metagpt/utils/index.html @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 24aabe8ae..d2cce3965 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -2,9 +2,10 @@ # -*- coding: utf-8 -*- """ @Time : 2023/7/4 10:53 -@Author : alexanderwu +@Author : alexanderwu alitrack @File : mermaid.py """ +import asyncio import subprocess from pathlib import Path @@ -12,48 +13,76 @@ from metagpt.config import CONFIG from metagpt.const import PROJECT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists +import os +import sys - -def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """suffix: png/svg/pdf :param mermaid_code: mermaid code :param output_file_without_suffix: output filename :param width: :param height: - :return: 0 if succed, -1 if failed + :return: 0 if succeed, -1 if failed """ # Write the Mermaid code to a temporary file + dir_name = os.path.dirname(output_file_without_suffix) + if dir_name and not os.path.exists(dir_name): + os.makedirs(dir_name) tmp = Path(f"{output_file_without_suffix}.mmd") tmp.write_text(mermaid_code, encoding="utf-8") + + engine = CONFIG.mermaid_engine.lower() + if engine == "nodejs": + if check_cmd_exists("mmdc") != 0: + logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") + return -1 + + for suffix in ["pdf", "svg", "png"]: + output_file = f"{output_file_without_suffix}.{suffix}" + # Call the `mmdc` command to convert the Mermaid code to a PNG + logger.info(f"Generating {output_file}..") - if check_cmd_exists("mmdc") != 0: - logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") - return -1 - - for suffix in ["pdf", "svg", "png"]: - output_file = f"{output_file_without_suffix}.{suffix}" - # Call the `mmdc` command to convert the Mermaid code to a PNG - logger.info(f"Generating {output_file}..") - - if CONFIG.puppeteer_config: - subprocess.run( - [ - CONFIG.mmdc, - "-p", - CONFIG.puppeteer_config, - "-i", - str(tmp), - "-o", - output_file, - "-w", - str(width), - "-H", - str(height), - ] + if CONFIG.puppeteer_config: + commands =[ + CONFIG.mmdc, + "-p", + CONFIG.puppeteer_config, + "-i", + str(tmp), + "-o", + output_file, + "-w", + str(width), + "-H", + str(height), + ] + else: + commands =[CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)] + process = await asyncio.create_subprocess_exec( + *commands, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE ) + + stdout, stderr = await process.communicate() + if stdout: + logger.info(stdout.decode()) + if stderr: + logger.error(stderr.decode()) + else: + + if engine =='playwright': + from metagpt.utils.mmdc_playwright import mermaid_to_file + return await mermaid_to_file(mermaid_code, output_file_without_suffix, width, height) + elif engine =='pyppeteer': + from metagpt.utils.mmdc_pyppeteer import mermaid_to_file + return await mermaid_to_file(mermaid_code, output_file_without_suffix, width, height) + elif engine =='ink': + from metagpt.utils.mmdc_ink import mermaid_to_file + return await mermaid_to_file(mermaid_code, output_file_without_suffix) else: - subprocess.run([CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) + logger.warning(f"Unsupported mermaid engine: {engine}") return 0 @@ -108,7 +137,9 @@ MMC2 = """sequenceDiagram SE-->>M: return summary""" + if __name__ == "__main__": - # logger.info(print_members(print_members)) - mermaid_to_file(MMC1, PROJECT_ROOT / "tmp/1.png") - mermaid_to_file(MMC2, PROJECT_ROOT / "tmp/2.png") + loop = asyncio.new_event_loop() + result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) + loop.close() diff --git a/metagpt/utils/mmdc_ink.py b/metagpt/utils/mmdc_ink.py new file mode 100644 index 000000000..3d91cde9d --- /dev/null +++ b/metagpt/utils/mmdc_ink.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/4 16:12 +@Author : alitrack +@File : mermaid.py +""" +import base64 +import os + +from aiohttp import ClientSession,ClientError +from metagpt.logs import logger + + +async def mermaid_to_file(mermaid_code, output_file_without_suffix): + """suffix: png/svg + :param mermaid_code: mermaid code + :param output_file_without_suffix: output filename without suffix + :return: 0 if succeed, -1 if failed + """ + encoded_string = base64.b64encode(mermaid_code.encode()).decode() + + for suffix in ["svg", "png"]: + output_file = f"{output_file_without_suffix}.{suffix}" + path_type = "svg" if suffix == "svg" else "img" + url = f"https://mermaid.ink/{path_type}/{encoded_string}" + async with ClientSession() as session: + try: + async with session.get(url) as response: + if response.status == 200: + text = await response.content.read() + with open(output_file, 'wb') as f: + f.write(text) + logger.info(f"Generating {output_file}..") + else: + logger.error(f"Failed to generate {output_file}") + return -1 + except ClientError as e: + logger.error(f"network error: {e}") + return -1 + return 0 diff --git a/metagpt/utils/mmdc_playwright.py b/metagpt/utils/mmdc_playwright.py new file mode 100644 index 000000000..bdbfd82ff --- /dev/null +++ b/metagpt/utils/mmdc_playwright.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/4 16:12 +@Author : Steven Lee +@File : mmdc_playwright.py +""" + +import os +from urllib.parse import urljoin +from playwright.async_api import async_playwright +from metagpt.logs import logger + +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int: + """ + Converts the given Mermaid code to various output formats and saves them to files. + + Args: + mermaid_code (str): The Mermaid code to convert. + output_file_without_suffix (str): The output file name without the file extension. + width (int, optional): The width of the output image in pixels. Defaults to 2048. + height (int, optional): The height of the output image in pixels. Defaults to 2048. + + Returns: + int: Returns 1 if the conversion and saving were successful, -1 otherwise. + """ + suffixes=['png', 'svg', 'pdf'] + __dirname = os.path.dirname(os.path.abspath(__file__)) + + async with async_playwright() as p: + browser = await p.chromium.launch() + device_scale_factor = 1.0 + context = await browser.new_context( + viewport={'width': width, 'height': height}, + device_scale_factor=device_scale_factor, + ) + page = await context.new_page() + + async def console_message(msg): + logger.info(msg.text) + page.on('console', console_message) + + try: + await page.set_viewport_size({'width': width, 'height': height}) + + mermaid_html_path = os.path.abspath( + os.path.join(__dirname, 'index.html')) + mermaid_html_url = urljoin('file:', mermaid_html_path) + await page.goto(mermaid_html_url) + await page.wait_for_load_state("networkidle") + + await page.wait_for_selector("div#container", state="attached") + mermaid_config = {} + background_color = "#ffffff" + my_css = "" + await page.evaluate(f'document.body.style.background = "{background_color}";') + + metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => { + const { mermaid, zenuml } = globalThis; + await mermaid.registerExternalDiagrams([zenuml]); + mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); + const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); + document.getElementById('container').innerHTML = svg; + const svgElement = document.querySelector('svg'); + svgElement.style.backgroundColor = backgroundColor; + + if (myCSS) { + const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); + style.appendChild(document.createTextNode(myCSS)); + svgElement.appendChild(style); + } + + }''', [mermaid_code, mermaid_config, my_css, background_color]) + + if 'svg' in suffixes : + svg_xml = await page.evaluate('''() => { + const svg = document.querySelector('svg'); + const xmlSerializer = new XMLSerializer(); + return xmlSerializer.serializeToString(svg); + }''') + logger.info(f"Generating {output_file_without_suffix}.svg..") + with open(f'{output_file_without_suffix}.svg', 'wb') as f: + f.write(svg_xml.encode('utf-8')) + + if 'png' in suffixes: + clip = await page.evaluate('''() => { + const svg = document.querySelector('svg'); + const rect = svg.getBoundingClientRect(); + return { + x: Math.floor(rect.left), + y: Math.floor(rect.top), + width: Math.ceil(rect.width), + height: Math.ceil(rect.height) + }; + }''') + await page.set_viewport_size({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height']}) + screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device') + logger.info(f"Generating {output_file_without_suffix}.png..") + with open(f'{output_file_without_suffix}.png', 'wb') as f: + f.write(screenshot) + if 'pdf' in suffixes: + pdf_data = await page.pdf(scale=device_scale_factor) + logger.info(f"Generating {output_file_without_suffix}.pdf..") + with open(f'{output_file_without_suffix}.pdf', 'wb') as f: + f.write(pdf_data) + return 0 + except Exception as e: + logger.error(e) + return -1 + finally: + await browser.close() diff --git a/metagpt/utils/mmdc_pyppeteer.py b/metagpt/utils/mmdc_pyppeteer.py new file mode 100644 index 000000000..7ec30fd12 --- /dev/null +++ b/metagpt/utils/mmdc_pyppeteer.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/4 16:12 +@Author : alitrack +@File : mmdc_pyppeteer.py +""" +import os +from urllib.parse import urljoin +from pyppeteer import launch +from metagpt.logs import logger +from metagpt.config import CONFIG + +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048)-> int: + """ + Converts the given Mermaid code to various output formats and saves them to files. + + Args: + mermaid_code (str): The Mermaid code to convert. + output_file_without_suffix (str): The output file name without the file extension. + width (int, optional): The width of the output image in pixels. Defaults to 2048. + height (int, optional): The height of the output image in pixels. Defaults to 2048. + + Returns: + int: Returns 1 if the conversion and saving were successful, -1 otherwise. + """ + suffixes = ['png', 'svg', 'pdf'] + __dirname = os.path.dirname(os.path.abspath(__file__)) + + + if CONFIG.pyppeteer_executable_path: + browser = await launch(headless=True, + executablePath=CONFIG.pyppeteer_executable_path, + args=['--disable-extensions',"--no-sandbox"] + ) + else: + logger.error("Please set the environment variable:PYPPETEER_EXECUTABLE_PATH.") + return -1 + page = await browser.newPage() + device_scale_factor = 1.0 + + async def console_message(msg): + logger.info(msg.text) + page.on('console', console_message) + + try: + await page.setViewport(viewport={'width': width, 'height': height, 'deviceScaleFactor': device_scale_factor}) + + mermaid_html_path = os.path.abspath( + os.path.join(__dirname, 'index.html')) + mermaid_html_url = urljoin('file:', mermaid_html_path) + await page.goto(mermaid_html_url) + + await page.querySelector("div#container") + mermaid_config = {} + background_color = "#ffffff" + my_css = "" + await page.evaluate(f'document.body.style.background = "{background_color}";') + + metadata = await page.evaluate('''async ([definition, mermaidConfig, myCSS, backgroundColor]) => { + const { mermaid, zenuml } = globalThis; + await mermaid.registerExternalDiagrams([zenuml]); + mermaid.initialize({ startOnLoad: false, ...mermaidConfig }); + const { svg } = await mermaid.render('my-svg', definition, document.getElementById('container')); + document.getElementById('container').innerHTML = svg; + const svgElement = document.querySelector('svg'); + svgElement.style.backgroundColor = backgroundColor; + + if (myCSS) { + const style = document.createElementNS('http://www.w3.org/2000/svg', 'style'); + style.appendChild(document.createTextNode(myCSS)); + svgElement.appendChild(style); + } + }''', [mermaid_code, mermaid_config, my_css, background_color]) + + if 'svg' in suffixes : + svg_xml = await page.evaluate('''() => { + const svg = document.querySelector('svg'); + const xmlSerializer = new XMLSerializer(); + return xmlSerializer.serializeToString(svg); + }''') + logger.info(f"Generating {output_file_without_suffix}.svg..") + with open(f'{output_file_without_suffix}.svg', 'wb') as f: + f.write(svg_xml.encode('utf-8')) + + if 'png' in suffixes: + clip = await page.evaluate('''() => { + const svg = document.querySelector('svg'); + const rect = svg.getBoundingClientRect(); + return { + x: Math.floor(rect.left), + y: Math.floor(rect.top), + width: Math.ceil(rect.width), + height: Math.ceil(rect.height) + }; + }''') + await page.setViewport({'width': clip['x'] + clip['width'], 'height': clip['y'] + clip['height'], 'deviceScaleFactor': device_scale_factor}) + screenshot = await page.screenshot(clip=clip, omit_background=True, scale='device') + logger.info(f"Generating {output_file_without_suffix}.png..") + with open(f'{output_file_without_suffix}.png', 'wb') as f: + f.write(screenshot) + if 'pdf' in suffixes: + pdf_data = await page.pdf(scale=device_scale_factor) + logger.info(f"Generating {output_file_without_suffix}.pdf..") + with open(f'{output_file_without_suffix}.pdf', 'wb') as f: + f.write(pdf_data) + return 0 + except Exception as e: + logger.error(e) + return -1 + finally: + await browser.close() + diff --git a/setup.py b/setup.py index a88f9de92..f9ae768e6 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ setup( "selenium": ["selenium>4", "webdriver_manager", "beautifulsoup4"], "search-google": ["google-api-python-client==2.94.0"], "search-ddg": ["duckduckgo-search==3.8.5"], + "pyppeteer": ["pyppeteer>=1.0.2"], }, cmdclass={ "install_mermaid": InstallMermaidCLI, diff --git a/startup.py b/startup.py index 611317fd4..e2a903c9b 100644 --- a/startup.py +++ b/startup.py @@ -1,11 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import asyncio -import platform + import fire -from metagpt.roles import Architect, Engineer, ProductManager -from metagpt.roles import ProjectManager, QaEngineer +from metagpt.roles import ( + Architect, + Engineer, + ProductManager, + ProjectManager, + QaEngineer, +) from metagpt.software_company import SoftwareCompany @@ -15,15 +20,17 @@ async def startup( n_round: int = 5, code_review: bool = False, run_tests: bool = False, - implement: bool = True + implement: bool = True, ): """Run a startup. Be a boss.""" company = SoftwareCompany() - company.hire([ - ProductManager(), - Architect(), - ProjectManager(), - ]) + company.hire( + [ + ProductManager(), + Architect(), + ProjectManager(), + ] + ) # if implement or code_review if implement or code_review: @@ -44,9 +51,9 @@ def main( idea: str, investment: float = 3.0, n_round: int = 5, - code_review: bool = False, + code_review: bool = True, run_tests: bool = False, - implement: bool = False + implement: bool = True, ): """ We are a software startup comprised of AI. By investing in us, @@ -58,12 +65,8 @@ def main( :param code_review: Whether to use code review. :return: """ - if platform.system() == "Windows": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - asyncio.run(startup(idea, investment, n_round, - code_review, run_tests, implement)) + asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) -if __name__ == '__main__': +if __name__ == "__main__": fire.Fire(main) - \ No newline at end of file diff --git a/tests/metagpt/actions/test_write_tutorial.py b/tests/metagpt/actions/test_write_tutorial.py new file mode 100644 index 000000000..683fee082 --- /dev/null +++ b/tests/metagpt/actions/test_write_tutorial.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# _*_ coding: utf-8 _*_ +""" +@Time : 2023/9/6 21:41:34 +@Author : Stitch-z +@File : test_write_tutorial.py +""" +from typing import Dict + +import pytest + +from metagpt.actions.write_tutorial import WriteDirectory, WriteContent + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("language", "topic"), + [("English", "Write a tutorial about Python")] +) +async def test_write_directory(language: str, topic: str): + ret = await WriteDirectory(language=language).run(topic=topic) + assert isinstance(ret, dict) + assert "title" in ret + assert "directory" in ret + assert isinstance(ret["directory"], list) + assert len(ret["directory"]) + assert isinstance(ret["directory"][0], dict) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("language", "topic", "directory"), + [("English", "Write a tutorial about Python", {"Introduction": ["What is Python?", "Why learn Python?"]})] +) +async def test_write_content(language: str, topic: str, directory: Dict): + ret = await WriteContent(language=language, directory=directory).run(topic=topic) + assert isinstance(ret, str) + assert list(directory.keys())[0] in ret + for value in list(directory.values())[0]: + assert value in ret diff --git a/tests/metagpt/roles/test_tutorial_assistant.py b/tests/metagpt/roles/test_tutorial_assistant.py new file mode 100644 index 000000000..945620cfc --- /dev/null +++ b/tests/metagpt/roles/test_tutorial_assistant.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# _*_ coding: utf-8 _*_ +""" +@Time : 2023/9/6 23:11:27 +@Author : Stitch-z +@File : test_tutorial_assistant.py +""" +import aiofiles +import pytest + +from metagpt.roles.tutorial_assistant import TutorialAssistant + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("language", "topic"), + [("Chinese", "Write a tutorial about Python")] +) +async def test_tutorial_assistant(language: str, topic: str): + topic = "Write a tutorial about MySQL" + role = TutorialAssistant(language=language) + msg = await role.run(topic) + filename = msg.content + title = filename.split("/")[-1].split(".")[0] + async with aiofiles.open(filename, mode="r") as reader: + content = await reader.read() + assert content.startswith(f"# {title}") \ No newline at end of file diff --git a/tests/metagpt/utils/test_file.py b/tests/metagpt/utils/test_file.py new file mode 100644 index 000000000..a9f1a353d --- /dev/null +++ b/tests/metagpt/utils/test_file.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# _*_ coding: utf-8 _*_ +""" +@Time : 2023/9/4 15:40:40 +@Author : Stitch-z +@File : test_file.py +""" +from pathlib import Path + +import aiofiles +import pytest + +from metagpt.utils.file import File + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("root_path", "filename", "content"), + [(Path("/code/MetaGPT/data/tutorial_docx/2023-09-07_17-05-20"), "test.md", "Hello World!")] +) +async def test_write_file(root_path: Path, filename: str, content: bytes): + full_file_name = await File.write(root_path=root_path, filename=filename, content=content.encode('utf-8')) + assert isinstance(full_file_name, Path) + assert root_path / filename == full_file_name + async with aiofiles.open(full_file_name, mode="r") as reader: + body = await reader.read() + assert body == content \ No newline at end of file