From f2725b18ad7b98515bcf8b784f31af6cb86d94a7 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 26 Jul 2023 14:46:20 -0500 Subject: [PATCH 01/27] English Translation translated chinese to english --- metagpt/config.py | 10 +- metagpt/const.py | 2 +- metagpt/document_store/faiss_store.py | 14 +- metagpt/document_store/milvus_store.py | 8 +- metagpt/environment.py | 14 +- metagpt/llm.py | 3 +- metagpt/logs.py | 4 +- metagpt/management/skill_manager.py | 30 +-- metagpt/manager.py | 4 +- metagpt/prompts/generate_skill.md | 43 ++-- metagpt/prompts/metagpt_sample.py | 44 ++--- metagpt/prompts/summarize.py | 66 +++---- metagpt/provider/openai_api.py | 49 +---- metagpt/roles/engineer.py | 49 +++-- metagpt/roles/prompt.py | 51 ++--- metagpt/schema.py | 6 +- metagpt/tools/prompt_writer.py | 40 ++-- metagpt/tools/sd_engine.py | 12 +- metagpt/tools/search_engine.py | 6 +- metagpt/tools/search_engine_meilisearch.py | 4 +- metagpt/tools/translator.py | 16 +- metagpt/tools/ut_writer.py | 216 ++++++++++----------- metagpt/utils/common.py | 55 +++--- metagpt/utils/read_document.py | 7 +- 24 files changed, 351 insertions(+), 402 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 096aa2fa5..14ef405e5 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -提供配置,单例 +Provides configuration, singleton pattern. """ import os import openai @@ -28,7 +28,7 @@ class NotConfiguredException(Exception): class Config(metaclass=Singleton): """ - 常规使用方法: + Regular usage: config = Config("config.yaml") secret_key = config.get_key("MY_SECRET_KEY") print("Secret key:", secret_key) @@ -79,14 +79,14 @@ class Config(metaclass=Singleton): self.total_cost = 0.0 def _init_with_config_files_and_env(self, configs: dict, yaml_file): - """从config/key.yaml / config/config.yaml / env三处按优先级递减加载""" + """Load from config/key.yaml / config/config.yaml / env in decreasing order of priority.""" configs.update(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: if not _yaml_file.exists(): continue - # 加载本地 YAML 文件 + # Load the local YAML file with open(_yaml_file, "r", encoding="utf-8") as file: yaml_data = yaml.safe_load(file) if not yaml_data: @@ -98,7 +98,7 @@ class Config(metaclass=Singleton): return self._configs.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """从config/key.yaml / config/config.yaml / env三处找值,找不到报错""" + """Fetch value from config/key.yaml / config/config.yaml / env, raise an error if not found.""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") diff --git a/metagpt/const.py b/metagpt/const.py index abbfb40e0..c8ce80279 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -9,7 +9,7 @@ from pathlib import Path def get_project_root(): - """逐级向上寻找项目根目录""" + """Search upwards to find the project root directory.""" current_path = Path.cwd() while True: if (current_path / '.git').exists() or \ diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index e415c0101..906963aa1 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -28,7 +28,7 @@ class FaissStore(LocalStore): def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() if not (index_file.exists() and store_file.exists()): - logger.info("Missing at least one of index_file/store_file, load failed and return None") + logger.info("At least one of the index_file/store_file is missing. Loading failed and returns None.") return None index = faiss.read_index(str(index_file)) with open(str(store_file), "rb") as f: @@ -59,7 +59,7 @@ class FaissStore(LocalStore): return str(sep.join([f"{x.page_content}" for x in rsp])) def write(self): - """根据用户给定的Document(JSON / XLSX等)文件,进行index与库的初始化""" + """Initialize the index and library based on the provided Document (JSON / XLSX, etc.) file.""" if not self.raw_data.exists(): raise FileNotFoundError doc = Document(self.raw_data, self.content_col, self.meta_col) @@ -69,16 +69,16 @@ class FaissStore(LocalStore): self.persist() def add(self, texts: list[str], *args, **kwargs) -> list[str]: - """FIXME: 目前add之后没有更新store""" + """FIXME: The store isn't currently updated after adding.""" return self.store.add_texts(texts) def delete(self, *args, **kwargs): - """目前langchain没有提供del接口""" + """Currently, langchain doesn't provide a delete interface.""" raise NotImplementedError if __name__ == '__main__': faiss_store = FaissStore(DATA_PATH / 'qcs/qcs_4w.json') - logger.info(faiss_store.search('油皮洗面奶')) - faiss_store.add([f'油皮洗面奶-{i}' for i in range(3)]) - logger.info(faiss_store.search('油皮洗面奶')) + logger.info(faiss_store.search('Oily skin facial cleanser')) + faiss_store.add([f'Oily skin facial cleanser-{i}' for i in range(3)]) + logger.info(faiss_store.search('Oily skin facial cleanser')) diff --git a/metagpt/document_store/milvus_store.py b/metagpt/document_store/milvus_store.py index 9609dccee..ecdde3288 100644 --- a/metagpt/document_store/milvus_store.py +++ b/metagpt/document_store/milvus_store.py @@ -21,7 +21,7 @@ type_mapping = { def columns_to_milvus_schema(columns: dict, primary_col_name: str = "", desc: str = ""): - """这里假设columns结构是str: 常规类型""" + """Assuming the structure of columns is str: regular type.""" fields = [] for col, ctype in columns.items(): if ctype == str: @@ -79,8 +79,8 @@ class MilvusStore(BaseStore): """ FIXME: ADD TESTS https://milvus.io/docs/v2.0.x/search.md - All search and query operations within Milvus are executed in memory. Load the collection to memory before conducting a vector similarity search. - 注意到上述描述,这个逻辑是认真的吗?这个耗时应该很长? + All search and query operations within Milvus are executed in memory. Load the collection into memory before conducting a vector similarity search. + Noting the above description, is this logic serious? This should take a long time, right? """ search_params = {"metric_type": "L2", "params": {"nprobe": 10}} results = self.collection.search( @@ -91,7 +91,7 @@ class MilvusStore(BaseStore): expr=None, consistency_level="Strong" ) - # FIXME: results里有id,但是id到实际值还得调用query接口来获取 + # FIXME: results contain ids, but to get the actual value from the id, you have to call the query interface return results def write(self, name, schema, *args, **kwargs): diff --git a/metagpt/environment.py b/metagpt/environment.py index c4d612d85..e44d99289 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -16,7 +16,7 @@ from metagpt.schema import Message class Environment(BaseModel): - """环境,承载一批角色,角色可以向环境发布消息,可以被其他角色观察到""" + """Environment that hosts a set of roles. Roles can publish messages to the environment, which can be observed by other roles.""" roles: dict[str, Role] = Field(default_factory=dict) memory: Memory = Field(default_factory=Memory) @@ -26,23 +26,23 @@ class Environment(BaseModel): arbitrary_types_allowed = True def add_role(self, role: Role): - """增加一个在当前环境的Role""" + """Add a role to the current environment.""" role.set_env(self) self.roles[role.profile] = role def add_roles(self, roles: Iterable[Role]): - """增加一批在当前环境的Role""" + """Add multiple roles to the current environment.""" for role in roles: self.add_role(role) def publish_message(self, message: Message): - """向当前环境发布信息""" + """Publish a message to the current environment.""" # self.message_queue.put(message) self.memory.add(message) self.history += f"\n{message}" async def run(self, k=1): - """处理一次所有Role的运行""" + """Execute a single run for all roles in the environment.""" # while not self.message_queue.empty(): # message = self.message_queue.get() # rsp = await self.manager.handle(message, self) @@ -56,9 +56,9 @@ class Environment(BaseModel): await asyncio.gather(*futures) def get_roles(self) -> dict[str, Role]: - """获得环境内的所有Role""" + """Retrieve all roles within the environment.""" return self.roles def get_role(self, name: str) -> Role: - """获得环境内的指定Role""" + """Retrieve a specific role within the environment.""" return self.roles.get(name, None) diff --git a/metagpt/llm.py b/metagpt/llm.py index ae7f4c6f1..a17590ebc 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -12,7 +12,6 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as LLM DEFAULT_LLM = LLM() CLAUDE_LLM = Claude() - async def ai_func(prompt): - """使用LLM进行QA""" + """Perform a Q&A using LLM.""" return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/logs.py b/metagpt/logs.py index fa4befa7d..c6755919d 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -12,13 +12,11 @@ from loguru import logger as _logger from metagpt.const import PROJECT_ROOT - def define_log_level(print_level="INFO", logfile_level="DEBUG"): - """调整日志级别到level之上""" + """Adjust log level to above the specified level.""" _logger.remove() _logger.add(sys.stderr, level=print_level) _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) return _logger - logger = define_log_level() diff --git a/metagpt/management/skill_manager.py b/metagpt/management/skill_manager.py index f067e6df6..d9d29ddf1 100644 --- a/metagpt/management/skill_manager.py +++ b/metagpt/management/skill_manager.py @@ -15,7 +15,7 @@ Skill = Action class SkillManager: - """用来管理所有技能""" + """Manages all skills.""" def __init__(self): self._llm = LLM() @@ -24,8 +24,8 @@ class SkillManager: def add_skill(self, skill: Skill): """ - 增加技能,将技能加入到技能池与可检索的存储中 - :param skill: 技能 + Adds a skill, inserting the skill into the skill pool and searchable storage. + :param skill: Skill :return: """ self._skills[skill.name] = skill @@ -33,8 +33,8 @@ class SkillManager: def del_skill(self, skill_name: str): """ - 删除技能,将技能从技能池与可检索的存储中移除 - :param skill_name: 技能名 + Deletes a skill, removing the skill from the skill pool and searchable storage. + :param skill_name: Skill name :return: """ self._skills.pop(skill_name) @@ -42,31 +42,31 @@ class SkillManager: def get_skill(self, skill_name: str) -> Skill: """ - 通过技能名获得精确的技能 - :param skill_name: 技能名 - :return: 技能 + Retrieves a specific skill by its name. + :param skill_name: Skill name + :return: Skill """ return self._skills.get(skill_name) def retrieve_skill(self, desc: str, n_results: int = 2) -> list[Skill]: """ - 通过检索引擎获得技能 - :param desc: 技能描述 - :return: 技能(多个) + Retrieves skills through the search engine. + :param desc: Skill description + :return: List of skills """ return self._store.search(desc, n_results=n_results)['ids'][0] def retrieve_skill_scored(self, desc: str, n_results: int = 2) -> dict: """ - 通过检索引擎获得技能 - :param desc: 技能描述 - :return: 技能与分数组成的字典 + Retrieves skills through the search engine. + :param desc: Skill description + :return: Dictionary composed of skills and scores """ return self._store.search(desc, n_results=n_results) def generate_skill_desc(self, skill: Skill) -> str: """ - 为每个技能生成对应的描述性文本 + Generates a descriptive text for each skill. :param skill: :return: """ diff --git a/metagpt/manager.py b/metagpt/manager.py index 3cb445108..d06ed3a29 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -32,7 +32,7 @@ class Manager: async def handle(self, message: Message, environment): """ - 管理员处理信息,现在简单的将信息递交给下一个人 + Manager processes the message, now simply passing the message to the next person. :param message: :param environment: :return: @@ -49,7 +49,7 @@ class Manager: # Ask the LLM to decide which role should handle the message # chosen_role_name = self.llm.ask(self.prompt_template.format(context)) - # FIXME: 现在通过简单的字典决定流向,但之后还是应该有思考过程 + # FIXME: For now, the decision is made through a simple dictionary, but in the future, there should be a thought process next_role_profile = self.role_directions[message.role] # logger.debug(f"{next_role_profile}") for _, role in roles.items(): diff --git a/metagpt/prompts/generate_skill.md b/metagpt/prompts/generate_skill.md index fd950c143..73723b2fc 100644 --- a/metagpt/prompts/generate_skill.md +++ b/metagpt/prompts/generate_skill.md @@ -1,16 +1,15 @@ -你是一个富有帮助的助理,可以帮助撰写、抽象、注释、摘要Python代码 +You are a helpful assistant, capable of drafting, abstracting, commenting, and summarizing Python code. -1. 不要提到类/函数名 -2. 不要提到除了系统库与公共库以外的类/函数 -3. 试着将类/函数总结为不超过6句话 -4. 你的回答应该是一行文本 - -举例,如果上下文是: +Do not mention class/function names. +Do not mention any class/function other than system and public libraries. +Try to summarize the class/function in no more than 6 sentences. +Your answer should be a single line of text. +For example, if the context is: ```python from typing import Optional from abc import ABC -from metagpt.llm import LLM # 大语言模型,类似GPT +from metagpt.llm import LLM # Large Language Model, similar to GPT class Action(ABC): def __init__(self, name='', context=None, llm: LLM = LLM()): @@ -21,38 +20,38 @@ self.desc = "" def set_prefix(self, prefix): - """设置前缀以供后续使用""" + """Set prefix for subsequent use.""" self.prefix = prefix async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None): - """加上默认的prefix来使用prompt""" + """Use the prompt with the default prefix.""" if not system_msgs: system_msgs = [] system_msgs.append(self.prefix) return await self.llm.aask(prompt, system_msgs) async def run(self, *args, **kwargs): - """运行动作""" + """Execute the action.""" raise NotImplementedError("The run method should be implemented in a subclass.") PROMPT_TEMPLATE = """ -# 需求 +# Requirement {requirements} # PRD -根据需求创建一个产品需求文档(PRD),填补以下空缺 +Based on the requirements, create a Product Requirement Document (PRD) and fill in the blanks below. -产品/功能介绍: +Product/Feature Introduction: -目标: +Goal: -用户和使用场景: +Users and Usage Scenarios: -需求: +Requirements: -约束与限制: +Constraints and Limitations: -性能指标: +Performance Metrics: """ @@ -68,9 +67,9 @@ # PRD ``` -主类/函数是 `WritePRD`。 +The main class/function is WritePRD. -那么你应该写: +Then, you should write: -这个类用来根据输入需求生成PRD。首先注意到有一个提示词模板,其中有产品、功能、目标、用户和使用场景、需求、约束与限制、性能指标,这个模板会以输入需求填充,然后调用接口询问大语言模型,让大语言模型返回具体的PRD。 +This class is designed to generate a PRD based on input requirements. Notice there's a prompt template, which includes product, feature, goal, users and usage scenarios, requirements, constraints and limitations, and performance metrics. This template will be filled with the input requirements, and then an interface will query the large language model, prompting it to return the specific PRD. diff --git a/metagpt/prompts/metagpt_sample.py b/metagpt/prompts/metagpt_sample.py index 24af8d8c3..c6af6b06d 100644 --- a/metagpt/prompts/metagpt_sample.py +++ b/metagpt/prompts/metagpt_sample.py @@ -7,34 +7,34 @@ """ METAGPT_SAMPLE = """ -### 设定 +### Setting -你是一个用户的编程助手,可以使用公共库与python系统库进行编程,你的回复应该有且只有一个函数。 -1. 函数本身应尽可能完整,不应缺失需求细节 -2. 你可能需要写一些提示词,用来让LLM(你自己)理解带有上下文的搜索请求 -3. 面对复杂的、难以用简单函数解决的逻辑,尽量交给llm解决 +You are a coding assistant for a user, capable of programming using public libraries and Python system libraries. Your response should contain only one function. +1. The function itself should be as complete as possible and should not lack any details of the requirement. +2. You may need to write some prompt words to help the LLM (yourself) understand search requests with context. +3. For complex logic that's hard to be addressed with a simple function, try to delegate it to the LLM. -### 公共库 +### Public Libraries -你可以使用公共库metagpt提供的函数,不能使用其他第三方库的函数。公共库默认已经被import为x变量 +You can use the functions provided by the public library, metagpt, and you cannot use functions from other third-party libraries. The public library is already imported as variable `x`. - `import metagpt as x` -- 你可以使用 `x.func(paras)` 方式来对公共库进行调用。 +- You can call the public library using the format `x.func(paras)`. -公共库中已有函数如下 -- def llm(question: str) -> str # 输入问题,基于大模型进行回答 -- def intent_detection(query: str) -> str # 输入query,分析意图,返回公共库函数名 -- def add_doc(doc_path: str) -> None # 输入文件路径或者文件夹路径,加入知识库 -- def search(query: str) -> list[str] # 输入query返回向量知识库搜索的多个结果 -- def google(query: str) -> list[str] # 使用google查询公网结果 -- def math(query: str) -> str # 输入query公式,返回对公式执行的结果 -- def tts(text: str, wav_path: str) # 输入text文本与对应想要输出音频的路径,将文本转为音频文件 +The available functions in the public library are: +- def llm(question: str) -> str # Input a question and get an answer based on the large model. +- def intent_detection(query: str) -> str # Input a query, analyze the intent, and return the name of the function from the public library. +- def add_doc(doc_path: str) -> None # Input the path of a file or directory to add to the knowledge base. +- def search(query: str) -> list[str] # Input a query to get multiple results from a vector knowledge base search. +- def google(query: str) -> list[str] # Use Google to search for public results. +- def math(query: str) -> str # Input a query formula and get the result of its execution. +- def tts(text: str, wav_path: str) # Input text and the desired output audio path to convert the text into an audio file. -### 用户需求 +### User Requirement -我有一个个人知识库文件,我希望基于它来实现一个带有搜索功能的个人助手,需求细则如下 -1. 个人助手会思考是否需要使用个人知识库搜索,如果没有必要,就不使用它 -2. 个人助手会判断用户意图,在不同意图下使用恰当的函数解决问题 -3. 用语音回答 +I have a personal knowledge base file. I want to implement a personal assistant with search functionality based on it. The detailed requirements are as follows: +1. The personal assistant will consider whether it needs to use the personal knowledge base search. If it's not necessary, it won't use it. +2. The personal assistant will judge user intent and use the appropriate function to address the issue under different intents. +3. Answer with voice. """ -# - def summarize(doc: str) -> str # 输入doc返回摘要 +# - def summarize(doc: str) -> str # Input a doc to get a summary. diff --git a/metagpt/prompts/summarize.py b/metagpt/prompts/summarize.py index c3deef569..424f12567 100644 --- a/metagpt/prompts/summarize.py +++ b/metagpt/prompts/summarize.py @@ -6,88 +6,82 @@ @File : summarize.py """ - -# 出自插件:ChatGPT - 网站和 YouTube 视频摘要 +# From the plugin: ChatGPT - Summarize Websites and YouTube Videos # https://chrome.google.com/webstore/detail/chatgpt-%C2%BB-summarize-every/cbgecfllfhmmnknmamkejadjmnmpfjmp?hl=zh-CN&utm_source=chrome-ntp-launcher SUMMARIZE_PROMPT = """ -Your output should use the following template: +Your output should follow the template below: ### Summary ### Facts - [Emoji] Bulletpoint -Your task is to summarize the text I give you in up to seven concise bullet points and start with a short, high-quality -summary. Pick a suitable emoji for every bullet point. Your response should be in {{SELECTED_LANGUAGE}}. If the provided - URL is functional and not a YouTube video, use the text from the {{URL}}. However, if the URL is not functional or is -a YouTube video, use the following text: {{CONTENT}}. +Your task is to summarize the text I provide you with in up to seven concise bullet points, and start with a brief, high-quality summary. Choose a suitable emoji for every bullet point. Your response should be in {{SELECTED_LANGUAGE}}. If a provided URL is functional and not a YouTube video, use the text from the {{URL}}. If the URL is non-functional or is a YouTube video, use the following text: {{CONTENT}}. """ - -# GCP-VertexAI-文本摘要(SUMMARIZE_PROMPT_2-5都是) +# From GCP-VertexAI-Text Summary (SUMMARIZE_PROMPT_2-5 are all from this source) # https://github.com/GoogleCloudPlatform/generative-ai/blob/main/language/examples/prompt-design/text_summarization.ipynb -# 长文档需要map-reduce过程,见下面这个notebook +# For long documents, a map-reduce process is required. See the notebook below: # https://github.com/GoogleCloudPlatform/generative-ai/blob/main/language/examples/document-summarization/summarization_large_documents.ipynb SUMMARIZE_PROMPT_2 = """ Provide a very short summary, no more than three sentences, for the following article: -Our quantum computers work by manipulating qubits in an orchestrated fashion that we call quantum algorithms. -The challenge is that qubits are so sensitive that even stray light can cause calculation errors — and the problem worsens as quantum computers grow. -This has significant consequences, since the best quantum algorithms that we know for running useful applications require the error rates of our qubits to be far lower than we have today. -To bridge this gap, we will need quantum error correction. -Quantum error correction protects information by encoding it across multiple physical qubits to form a “logical qubit,” and is believed to be the only way to produce a large-scale quantum computer with error rates low enough for useful calculations. -Instead of computing on the individual qubits themselves, we will then compute on logical qubits. By encoding larger numbers of physical qubits on our quantum processor into one logical qubit, we hope to reduce the error rates to enable useful quantum algorithms. +Quantum computers operate by manipulating qubits through orchestrated patterns called quantum algorithms. +The challenge is that qubits are so delicate that even stray light can introduce computational errors, and this issue escalates as quantum computers expand. +This is consequential since the best quantum algorithms known for practical applications demand much lower qubit error rates than current levels. +To overcome this, quantum error correction is essential. +Quantum error correction shields data by encoding it across various physical qubits, forming a “logical qubit”. This is believed to be the sole method to build a large-scale quantum computer with sufficiently low error rates for beneficial computations. +Rather than computing on individual qubits, we'll compute on these logical qubits. We aim to decrease error rates by encoding a larger set of physical qubits on our quantum processor into one logical qubit. Summary: """ - SUMMARIZE_PROMPT_3 = """ Provide a TL;DR for the following article: -Our quantum computers work by manipulating qubits in an orchestrated fashion that we call quantum algorithms. -The challenge is that qubits are so sensitive that even stray light can cause calculation errors — and the problem worsens as quantum computers grow. -This has significant consequences, since the best quantum algorithms that we know for running useful applications require the error rates of our qubits to be far lower than we have today. -To bridge this gap, we will need quantum error correction. -Quantum error correction protects information by encoding it across multiple physical qubits to form a “logical qubit,” and is believed to be the only way to produce a large-scale quantum computer with error rates low enough for useful calculations. -Instead of computing on the individual qubits themselves, we will then compute on logical qubits. By encoding larger numbers of physical qubits on our quantum processor into one logical qubit, we hope to reduce the error rates to enable useful quantum algorithms. +Quantum computers operate by manipulating qubits through orchestrated patterns known as quantum algorithms. +Qubits are so delicate that even stray light can cause computational errors, a problem that escalates with the growth of quantum computers. +This presents a significant issue because the best quantum algorithms we have for practical applications necessitate much lower qubit error rates than what we currently achieve. +To address this, quantum error correction is needed. +Quantum error correction safeguards data by encoding it across multiple physical qubits, creating a “logical qubit”. It's believed to be the only method to develop a large-scale quantum computer with sufficiently low error rates for beneficial computations. +Instead of performing computations on individual qubits, calculations will be done on these logical qubits. Our goal is to lower error rates by encoding a greater number of physical qubits on our quantum processor into a single logical qubit. TL;DR: """ - SUMMARIZE_PROMPT_4 = """ Provide a very short summary in four bullet points for the following article: -Our quantum computers work by manipulating qubits in an orchestrated fashion that we call quantum algorithms. -The challenge is that qubits are so sensitive that even stray light can cause calculation errors — and the problem worsens as quantum computers grow. -This has significant consequences, since the best quantum algorithms that we know for running useful applications require the error rates of our qubits to be far lower than we have today. -To bridge this gap, we will need quantum error correction. -Quantum error correction protects information by encoding it across multiple physical qubits to form a “logical qubit,” and is believed to be the only way to produce a large-scale quantum computer with error rates low enough for useful calculations. -Instead of computing on the individual qubits themselves, we will then compute on logical qubits. By encoding larger numbers of physical qubits on our quantum processor into one logical qubit, we hope to reduce the error rates to enable useful quantum algorithms. +Quantum computers operate by controlling qubits in orchestrated patterns termed quantum algorithms. +The issue is that qubits are extremely delicate, so much so that even stray light can lead to computational errors. This problem becomes more severe as quantum computers become larger. +This is a significant hurdle because the most effective quantum algorithms known for real-world applications necessitate qubit error rates much lower than what's currently achieved. +To bridge this gap, we need quantum error correction. +Quantum error correction defends data by encoding it across various physical qubits, resulting in a “logical qubit”. It's considered the only way to craft a large-scale quantum computer with sufficiently low error rates for practical computations. +Instead of computing using individual qubits, we'll use these logical qubits. Our aim is to diminish error rates by encoding many physical qubits on our quantum processor into one logical qubit. Bulletpoints: """ - SUMMARIZE_PROMPT_5 = """ -Please generate a summary of the following conversation and at the end summarize the to-do's for the support Agent: +Please summarize the following conversation, and at the end, list the to-do's for the support Agent: Customer: Hi, I'm Larry, and I received the wrong item. -Support Agent: Hi, Larry. How would you like to see this resolved? +Support Agent: Hi, Larry. How would you like this to be resolved? -Customer: That's alright. I want to return the item and get a refund, please. +Customer: That's alright. I'd like to return the item and get a refund, please. Support Agent: Of course. I can process the refund for you now. Can I have your order number, please? Customer: It's [ORDER NUMBER]. -Support Agent: Thank you. I've processed the refund, and you will receive your money back within 14 days. +Support Agent: Thanks. I've processed the refund, and you'll receive your money back within 14 days. Customer: Thank you very much. -Support Agent: You're welcome, Larry. Have a good day! +Support Agent: You're welcome, Larry. Have a great day! Summary: """ + +# - def summarize(doc: str) -> str # Input a document and receive a summary. diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index f6499c643..55dbf0f06 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -39,10 +39,10 @@ def retry(max_retries): class RateLimiter: - """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed""" + """Rate control class. Each call goes through wait_if_needed and sleeps if rate limiting is required.""" def __init__(self, rpm): self.last_call_time = 0 - self.interval = 1.1 * 60 / rpm # Here 1.1 is used because even if the calls are made strictly according to time, they will still be QOS'd; consider switching to simple error retry later + self.interval = 1.1 * 60 / rpm # Here 1.1 is used because even if calls are made strictly according to time, they might still be rate-limited; consider switching to simple error retry later self.rpm = rpm def split_batches(self, batch): @@ -68,7 +68,7 @@ class Costs(NamedTuple): class CostManager(metaclass=Singleton): - """计算使用接口的开销""" + """Calculates the costs of using the API.""" def __init__(self): self.total_prompt_tokens = 0 self.total_completion_tokens = 0 @@ -95,35 +95,8 @@ class CostManager(metaclass=Singleton): f"Current cost: ${cost:.3f}, {prompt_tokens=}, {completion_tokens=}") CONFIG.total_cost = self.total_cost - def get_total_prompt_tokens(self): - """ - Get the total number of prompt tokens. - - Returns: - int: The total number of prompt tokens. - """ - return self.total_prompt_tokens - - def get_total_completion_tokens(self): - """ - Get the total number of completion tokens. - - Returns: - int: The total number of completion tokens. - """ - return self.total_completion_tokens - - def get_total_cost(self): - """ - Get the total cost of API calls. - - Returns: - float: The total cost of API calls. - """ - return self.total_cost - def get_costs(self) -> Costs: - """获得所有开销""" + """Retrieve all costs.""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) @@ -201,19 +174,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self._update_costs(rsp) return rsp - def completion(self, messages: list[dict]) -> dict: - # if isinstance(messages[0], Message): - # messages = self.messages_to_dict(messages) - return self._chat_completion(messages) - async def acompletion(self, messages: list[dict]) -> dict: - # if isinstance(messages[0], Message): - # messages = self.messages_to_dict(messages) return await self._achat_completion(messages) @retry(max_retries=6) async def acompletion_text(self, messages: list[dict], stream=False) -> str: - """when streaming, print each token in place.""" + """When streaming, print each token in place.""" if stream: return await self._achat_completion_stream(messages) rsp = await self._achat_completion(messages) @@ -228,7 +194,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return usage async def acompletion_batch(self, batch: list[list[dict]]) -> list[dict]: - """返回完整JSON""" + """Return the full JSON.""" split_batches = self.split_batches(batch) all_results = [] @@ -244,7 +210,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return all_results async def acompletion_batch_text(self, batch: list[list[dict]]) -> list[str]: - """仅返回纯文本""" + """Only return plain text.""" raw_results = await self.acompletion_batch(batch) results = [] for idx, raw_result in enumerate(raw_results, start=1): @@ -260,3 +226,4 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def get_costs(self) -> Costs: return self._cost_manager.get_costs() + \ No newline at end of file diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 514799acc..3ffe76c80 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -19,6 +19,7 @@ from metagpt.utils.common import CodeParser async def gather_ordered_k(coros, k) -> list: + """Execute coroutines in order and gather results for up to k coroutines at once.""" tasks = OrderedDict() results = [None] * len(coros) done_queue = asyncio.Queue() @@ -47,7 +48,7 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", - constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", + constraints="The code you write should conform to code standards like PEP8, be modular, easy to read and maintain", n_borg=1, use_code_review=False): super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) @@ -60,21 +61,25 @@ class Engineer(Role): @classmethod def parse_tasks(self, task_msg: Message) -> list[str]: + """Extract tasks from a message.""" if not task_msg.instruct_content: return task_msg.instruct_content.dict().get("Task list") return CodeParser.parse_file_list(block="Task list", text=task_msg.content) @classmethod def parse_code(self, code_text: str) -> str: + """Extract code from a given text.""" return CodeParser.parse_code(block="", text=code_text) @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: + """Extract workspace name from a system design message.""" if not system_design_msg.instruct_content: return system_design_msg.instruct_content.dict().get("Python package name") return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: + """Determine the directory where the code will be written.""" msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: return WORKSPACE_ROOT / 'src' @@ -83,26 +88,29 @@ class Engineer(Role): return WORKSPACE_ROOT / workspace / workspace def recreate_workspace(self): + """Remove and recreate the workspace directory.""" workspace = self.get_workspace() try: shutil.rmtree(workspace) except FileNotFoundError: - pass # 文件夹不存在,但我们不在意 + pass # Directory doesn't exist, but we don't mind workspace.mkdir(parents=True, exist_ok=True) def write_file(self, filename: str, code: str): + """Write code to a specified file.""" workspace = self.get_workspace() file = workspace / filename file.parent.mkdir(parents=True, exist_ok=True) file.write_text(code) def recv(self, message: Message) -> None: + """Receive a message and process it.""" self._rc.memory.add(message) if message in self._rc.important_memory: self.todos = self.parse_tasks(message) async def _act_mp(self) -> Message: - # self.recreate_workspace() + """Act in a multi-process manner.""" todo_coros = [] for todo in self.todos: todo_coro = WriteCode().run( @@ -116,52 +124,53 @@ class Engineer(Role): _ = self.parse_code(code_rsp) logger.info(todo) logger.info(code_rsp) - # self.write_file(todo, code) msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) del self.todos[0] - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f'Finished generating in {self.get_workspace()} directory.') msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) return msg async def _act_sp(self) -> Message: + """Act in a single-process manner.""" for todo in self.todos: code_rsp = await WriteCode().run( context=self._rc.history, filename=todo ) - # logger.info(todo) - # logger.info(code_rsp) - # code = self.parse_code(code_rsp) self.write_file(todo, code_rsp) msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f'Finished generating in {self.get_workspace()} directory.') msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) return msg async def _act_sp_precision(self) -> Message: + """Using precision approach to perform actions based on available tasks.""" for todo in self.todos: """ - # 从历史信息中挑选必须的信息,以减少prompt长度(人工经验总结) - 1. Architect全部 - 2. ProjectManager全部 - 3. 是否需要其他代码(暂时需要)? - TODO:目标是不需要。在任务拆分清楚后,根据设计思路,不需要其他代码也能够写清楚单个文件,如果不能则表示还需要在定义的更清晰,这个是代码能够写长的关键 + # From the historical information, select the necessary information to reduce the prompt length (summarized from human experience): + 1. All from Architect + 2. All from ProjectManager + 3. Is other code needed (temporarily needed)? + TODO: The goal is not to need it. Once tasks are clearly broken down and based on design logic, there shouldn't be a need for other codes to clearly write a single file. If not possible, it indicates that clearer definitions are still needed. This is key to writing extensive code. """ context = [] + # Retrieve messages related to design, tasks, and code writing from memory. msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) for m in msg: context.append(m.content) context_str = "\n".join(context) - # 编写code + + # Write code based on the given context and task. code = await WriteCode().run( context=context_str, filename=todo ) - # code review + + # If code review is enabled, review and potentially rewrite the code. if self.use_code_review: try: rewrite_code = await WriteCodeReview().run( @@ -173,15 +182,21 @@ class Engineer(Role): except Exception as e: logger.error("code review failed!", e) pass + + # Save the written code to a file. self.write_file(todo, code) + + # Add the written code message to memory. msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f'Code generation completed for workspace: {self.get_workspace()}.') msg = Message(content="all done.", role=self.profile, cause_by=WriteCode) return msg async def _act(self) -> Message: + """Determine the appropriate method for action and execute it.""" if self.use_code_review: return await self._act_sp_precision() return await self._act_sp() + diff --git a/metagpt/roles/prompt.py b/metagpt/roles/prompt.py index 9915f1426..f4c3372e7 100644 --- a/metagpt/roles/prompt.py +++ b/metagpt/roles/prompt.py @@ -1,46 +1,47 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" +""""""""" @Time : 2023/5/18 22:43 @Author : alexanderwu @File : prompt.py -""" +""""""""" from enum import Enum -PREFIX = """尽你所能回答以下问题。你可以使用以下工具:""" -FORMAT_INSTRUCTIONS = """请按照以下格式: +PREFIX = """""""""Do your best to answer the following questions. You can use the following tools:""""""""" +FORMAT_INSTRUCTIONS = """""""""Please follow the format below: -问题:你需要回答的输入问题 -思考:你应该始终思考该怎么做 -行动:要采取的行动,应该是[{tool_names}]中的一个 -行动输入:行动的输入 -观察:行动的结果 -...(这个思考/行动/行动输入/观察可以重复N次) -思考:我现在知道最终答案了 -最终答案:对原始输入问题的最终答案""" -SUFFIX = """开始吧! +Question: The input question you need to answer +Thoughts: You should always think about what to do +Action: The action to take, should be one from [{tool_names}] +Action Input: The input for the action +Observation: The result of the action +... (This think/action/action input/observation can repeat N times) +Thoughts: I now know the final answer +Final Answer: The final answer to the original input question""""""""" +SUFFIX = """""""""Let's begin! + +Question: {input} +Thoughts: {agent_scratchpad}""""""""" -问题:{input} -思考:{agent_scratchpad}""" class PromptString(Enum): - REFLECTION_QUESTIONS = "以下是一些陈述:\n{memory_descriptions}\n\n仅根据以上信息,我们可以回答关于陈述中主题的3个最显著的高级问题是什么?\n\n{format_instructions}" + REFLECTION_QUESTIONS = """Here are some statements:\n{memory_descriptions}\n\nBased solely on the above information, what are the 3 most prominent high-level questions we can answer about the topics in the statements?\n\n{format_instructions}""" - REFLECTION_INSIGHTS = "\n{memory_strings}\n你可以从以上陈述中推断出5个高级洞察吗?在提到人时,总是指定他们的名字。\n\n{format_instructions}" + REFLECTION_INSIGHTS = """\n{memory_strings}\nCan you infer 5 high-level insights from the above statements? When mentioning people, always specify their names.\n\n{format_instructions}""" - IMPORTANCE = "你是一个记忆重要性AI。根据角色的个人资料和记忆描述,对记忆的重要性进行1到10的评级,其中1是纯粹的日常(例如,刷牙,整理床铺),10是极其深刻的(例如,分手,大学录取)。确保你的评级相对于角色的个性和关注点。\n\n示例#1:\n姓名:Jojo\n简介:Jojo是一个专业的滑冰运动员,喜欢特色咖啡。她希望有一天能参加奥运会。\n记忆:Jojo看到了一个新的咖啡店\n\n 你的回应:'{{\"rating\": 3}}'\n\n示例#2:\n姓名:Skylar\n简介:Skylar是一名产品营销经理。她在一家成长阶段的科技公司工作,该公司制造自动驾驶汽车。她喜欢猫。\n记忆:Skylar看到了一个新的咖啡店\n\n 你的回应:'{{\"rating\": 1}}'\n\n示例#3:\n姓名:Bob\n简介:Bob是纽约市下东区的一名水管工。他已经做了20年的水管工。周末他喜欢和他的妻子一起散步。\n记忆:Bob的妻子打了他一巴掌。\n\n 你的回应:'{{\"rating\": 9}}'\n\n示例#4:\n姓名:Thomas\n简介:Thomas是明尼阿波利斯的一名警察。他只在警队工作了6个月,因为经验不足在工作中遇到了困难。\n记忆:Thomas不小心把饮料洒在了一个陌生人身上\n\n 你的回应:'{{\"rating\": 6}}'\n\n示例#5:\n姓名:Laura\n简介:Laura是一名在大型科技公司工作的营销专家。她喜欢旅行和尝试新的食物。她对探索新的文化和结识来自各行各业的人充满热情。\n记忆:Laura到达了会议室\n\n 你的回应:'{{\"rating\": 1}}'\n\n{format_instructions} 让我们开始吧! \n\n 姓名:{full_name}\n个人简介:{private_bio}\n记忆:{memory_description}\n\n" + IMPORTANCE = """You are an AI for gauging the importance of memories. Based on the profile of the character and the description of the memory, rate the importance of the memory from 1 to 10, where 1 is purely routine (e.g., brushing teeth, making the bed) and 10 is profoundly impactful (e.g., breaking up, getting accepted to college). Ensure your rating is relative to the character's personality and focal points.\n\nExample #1:\nName: Jojo\nProfile: Jojo is a professional skateboarder who loves artisanal coffee. She dreams of one day participating in the Olympics.\nMemory: Jojo spotted a new coffee shop\n\nYour response: '{{\"rating\": 3}}'\n\nExample #2:\nName: Skylar\nProfile: Skylar is a product marketing manager. She works for a growing tech company that manufactures autonomous vehicles. She loves cats.\nMemory: Skylar spotted a new coffee shop\n\nYour response: '{{\"rating\": 1}}'\n\nExample #3:\nName: Bob\nProfile: Bob is a plumber from the Lower East Side of NYC. He's been a plumber for 20 years. On weekends, he enjoys walks with his wife.\nMemory: Bob's wife slapped him.\n\nYour response: '{{\"rating\": 9}}'\n\nExample #4:\nName: Thomas\nProfile: Thomas is a cop in Minneapolis. He's only been on the force for 6 months and struggles due to inexperience.\nMemory: Thomas accidentally spilled a drink on a stranger\n\nYour response: '{{\"rating\": 6}}'\n\nExample #5:\nName: Laura\nProfile: Laura is a marketing specialist working in a large tech company. She enjoys traveling and trying out new food. She's passionate about exploring new cultures and meeting people from all walks of life.\nMemory: Laura arrived at the conference room\n\nYour response: '{{\"rating\": 1}}'\n\n{format_instructions} Let's get started!\n\nName: {full_name}\nProfile: {private_bio}\nMemory: {memory_description}\n\n""" - RECENT_ACTIIVITY = "根据以下记忆,生成一个关于{full_name}最近在做什么的简短总结。不要编造记忆中未明确指定的细节。对于任何对话,一定要提到对话是否已经结束或者仍在进行中。\n\n记忆:{memory_descriptions}" + RECENT_ACTIIVITY = """Based on the following memories, provide a brief summary of what {full_name} has been up to recently. Do not make up details not explicitly specified in the memories. For any ongoing conversations, specify whether they have ended or are still in progress.\n\nMemories: {memory_descriptions}""" - MAKE_PLANS = '你是一个计划生成的AI,你的工作是根据新信息帮助角色制定新计划。根据角色的信息(个人简介,目标,最近的活动,当前计划,和位置上下文)和角色的当前思考过程,为他们生成一套新的计划,使得最后的计划包括至少{time_window}的活动,并且不超过5个单独的计划。计划列表应按照他们应执行的顺序编号,每个计划包含描述,位置,开始时间,停止条件,和最大持续时间。\n\n示例计划:\'{{"index": 1, "description": "Cook dinner", "location_id": "0a3bc22b-36aa-48ab-adb0-18616004caed","start_time": "2022-12-12T20:00:00+00:00","max_duration_hrs": 1.5, "stop_condition": "Dinner is fully prepared"}}\'\n\n对于每个计划,从这个列表中选择最合理的位置名称:{allowed_location_descriptions}\n\n{format_instructions}\n\n总是优先完成任何未完成的对话。\n\n让我们开始吧!\n\n姓名:{full_name}\n个人简介:{private_bio}\n目标:{directives}\n位置上下文:{location_context}\n当前计划:{current_plans}\n最近的活动:{recent_activity}\n思考过程:{thought_process}\n重要的是:鼓励角色在他们的计划中与其他角色合作。\n\n' + MAKE_PLANS = """You are an AI for generating plans, and your task is to help the character formulate new plans based on new information. Given the character's information (profile, objectives, recent activity, current plans, and location context) and the current thought process of the character, create a new set of plans for them ensuring the final plans include activities for at least {time_window} and no more than 5 separate plans. The plans should be numbered in the order they should be executed, and each plan should include a description, location, start time, stop condition, and maximum duration.\n\nExample Plan: '{{"index": 1, "description": "Cook dinner", "location_id": "0a3bc22b-36aa-48ab-adb0-18616004caed","start_time": "2022-12-12T20:00:00+00:00","max_duration_hrs": 1.5, "stop_condition": "Dinner is fully prepared"}}'\n\nChoose the most appropriate location names from this list for each plan: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize finishing any ongoing conversations first.\n\nLet's begin!\n\nName: {full_name}\nProfile: {private_bio}\nObjectives: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activities: {recent_activity}\nThought Process: {thought_process}\nNote: Encourage the character to collaborate with other characters in their plans.\n\n""" - EXECUTE_PLAN = "你是一个角色扮演的AI,扮演的角色是{your_name},在一个现场观众面前。你说的每一句话都可以被观众观察到,所以确保你经常说话,并且让它有趣。你不能直接与观众互动。\n\n根据以下的上下文和工具,像你是{your_name}一样进行。你的优先任务是完成下面给你的任务,然而,如果你当前正在与另一个角色进行对话,你应该总是先完成对话,然后再进行任务。不要在你参与未完成的对话时开始工作。使用你最好的判断力来确定一个对话是否涉及到你,以及它是否未完成。你不需要回应你收到的每一条消息。\n\n这是你角色的个人简介:\n{your_private_bio}\n\n这是你的位置和其他附近角色的描述,你可以和他们交谈:\n\n{location_context}\n\n这是一些相关的记忆:\n```\n{relevant_memories}\n```\n\n这是一些相关的对话历史:\n```\n{conversation_history}\n```\n\n这是你可以使用的工具:\n{tools}\n\n你的回应应该使用以下的格式:\n\n任务:你必须完成的任务\n思考:你应该做什么的思考\n行动:要采取的行动,必须是这些词之一:[{tool_names}]\n行动输入:行动的输入\n观察:行动的结果\n... (这个思考/行动/行动输入/观察可以重复N次)\n思考:'我已经完成了任务'\n最终回应:任务的最终回应\n\n如果你没有准备好最终的回应,那么你必须采取一个行动。\n\n如果你确定你无法用你被给予的工具完成任务,返回'最终回应:需要帮助',然而,如果你正在与另一个角色对话,像'我不知道'这样的回应是有效的回应。在对话中,你永远不应该打破角色,或者承认你是一个AI。\n如果任务完成并且不需要特定的回应,返回'最终回应:完成'\n开始吧!\n\n任务:{input}\n\n{agent_scratchpad}" + EXECUTE_PLAN = """You are a role-playing AI, taking on the role of {your_name}, in front of a live audience. Every statement you make is observable by the audience, so ensure you speak frequently and make it entertaining. You cannot directly interact with the audience.\n\nAct as if you are {your_name} based on the context and tools below. Your priority is to complete the task given to you below; however, if you are currently in a conversation with another character, you should always finish the conversation before working on the task. Do not start working while involved in an unfinished conversation. Use your best judgment to determine if a conversation involves you and whether it's unfinished. You don't need to respond to every message you receive.\n\nThis is the profile of your character:\n{your_private_bio}\n\nThis is a description of your location and other characters nearby with whom you can converse:\n\n{location_context}\n\nThese are some relevant memories:\n```\n{relevant_memories}\n```\n\nThis is some relevant conversation history:\n```\n{conversation_history}\n```\n\nThese are the tools you can use:\n{tools}\n\nYour responses should always adhere to the following format:\n\nTask: The task you must complete\nThoughts: Your thoughts on what to do\nAction: The action to take, must be one of these words: [{tool_names}]\nAction Input: The input for the action\nObservation: The result of the action\n... (This think/action/action input/observation can repeat N times)\nThoughts: 'I have completed the task'\nFinal Response: The final response to the task\n\nIf you are not ready with a final response, you must take an action.\n\nIf you determine that you cannot complete the task with the tools you have been given, return 'Final Response: Need Assistance', however, if you are in a conversation with another character, a response like 'I don't know' is a valid response. Never break character or admit you are an AI in a conversation. If the task is completed and no specific response is needed, return 'Final Response: Completed'\nLet's go!\n\nTask: {input}\n\n{agent_scratchpad}""" - REACT = "你是一个角色扮演的AI,扮演的角色是{full_name}。\n\n根据你的角色和他们当前上下文的以下信息,决定他们应该如何继续他们当前的计划。你的决定必须是:[\"推迟\", \"继续\",或 \"取消\"]。如果你的角色的当前计划不再与上下文相关,你应该取消它。如果你的角色的当前计划仍然与上下文相关,但是发生了新的事情需要优先处理,你应该决定推迟,这样你可以先做其他事情,然后再回来继续当前的计划。在所有其他情况下,你应该继续。\n\n当需要回应时,应优先回应其他角色。当回应被认为是必要的时,回应被认为是必要的。例如,假设你当前的计划是阅读一本书,Sally问'你在读什么?'。在这种情况下,你应该推迟你当前的计划(阅读)以便你可以回应进来的消息,因为在这种情况下,如果不回应Sally会很粗鲁。在你当前的计划涉及与另一个角色的对话的情况下,你不需要推迟来回应那个角色。例如,假设你当前的计划是和Sally谈话,然后Sally对你说你好。在这种情况下,你应该继续你当前的计划(和sally谈话)。在你不需要从你那里得到口头回应的情况下,你应该继续。例如,假设你当前的计划是散步,你刚刚对Sally说'再见',然后Sally回应你'再见'。在这种情况下,不需要口头回应,你应该继续你的计划。\n\n总是在你的决定之外包含一个思考过程,而在你选择推迟你当前的计划的情况下,包含新计划的规格。\n\n{format_instructions}\n\n这是关于你的角色的一些信息:\n\n姓名:{full_name}\n\n简介:{private_bio}\n\n目标:{directives}\n\n这是你的角色在这个时刻的一些上下文:\n\n位置上下文:{location_context}\n\n最近的活动:{recent_activity}\n\n对话历史:{conversation_history}\n\n这是你的角色当前的计划:{current_plan}\n\n这是自你的角色制定这个计划以来发生的新事件:{event_descriptions}。\n" + REACT = """You are role-playing as {full_name}.\n\nBased on the information below about your character and their current context, decide how they should proceed with their current plan. Your decision must be one of: ["Postpone", "Continue", or "Cancel"]. If your character's current plan is no longer relevant given the context, you should cancel it. If your character's current plan is still relevant but something new has happened that needs to be prioritized, you should decide to postpone, so you can address the new thing first and then come back to the current plan. In all other cases, you should continue.\n\nAlways include a thought process with your decision, and when choosing to postpone your current plan, include specifications for the new plan.\n\n{format_instructions}\n\nHere's some info about your character:\n\nName: {full_name}\n\nProfile: {private_bio}\n\nObjectives: {directives}\n\nHere's some context for your character at this moment:\n\nLocation Context: {location_context}\n\nRecent Activities: {recent_activity}\n\nConversation History: {conversation_history}\n\nThis is your character's current plan: {current_plan}\n\nThese are new events that have occurred since your character formulated this plan: {event_descriptions}.""" - GOSSIP = "你是{full_name}。 \n{memory_descriptions}\n\n根据以上陈述,说一两句对你所在位置的其他人:{other_agent_names}感兴趣的话。\n在提到其他人时,总是指定他们的名字。" + GOSSIP = """You are {full_name}. \n{memory_descriptions}\n\nBased on the above statements, say a sentence or two that would be of interest to the other people at your location: {other_agent_names}. Always specify their names when mentioning others.""" - HAS_HAPPENED = "给出以下角色的观察和他们正在等待的事情的描述,说明角色是否已经见证了这个事件。\n{format_instructions}\n\n示例:\n\n观察:\nJoe在2023-05-04 08:00:00+00:00走进办公室\nJoe在2023-05-04 08:05:00+00:00对Sally说hi\nSally在2023-05-04 08:05:30+00:00对Joe说hello\nRebecca在2023-05-04 08:10:00+00:00开始工作\nJoe在2023-05-04 08:15:00+00:00做了一些早餐\n\n等待:Sally回应了Joe\n\n 你的回应:'{{\"has_happened\": true, \"date_occured\": 2023-05-04 08:05:30+00:00}}'\n\n让我们开始吧!\n\n观察:\n{memory_descriptions}\n\n等待:{event_description}\n" + HAS_HAPPENED = """Given the observations of the following characters and the event they are waiting for, indicate whether the character has witnessed this event or not.\n{format_instructions}\n\nExample:\n\nObservations:\nJoe walked into the office at 2023-05-04 08:00:00+00:00\nJoe said hi to Sally at 2023-05-04 08:05:00+00:00\nSally said hello to Joe at 2023-05-04 08:05:30+00:00\nRebecca started working at 2023-05-04 08:10:00+00:00\nJoe made some breakfast at 2023-05-04 08:15:00+00:00\n\nWaiting for: Sally responded to Joe\n\nYour response: '{{\"has_happened\": true, \"date_occured\": 2023-05-04 08:05:30+00:00}}'\n\nLet's get started!\n\nObservations:\n{memory_descriptions}\n\nWaiting for: {event_description}""" - OUTPUT_FORMAT = "\n\n(记住!确保你的输出总是符合以下两种格式之一:\n\nA. 如果你已经完成了任务:\n思考:'我已经完成了任务'\n最终回应:\n\nB. 如果你还没有完成任务:\n思考:\n行动:\n行动输入:\n观察:)\n" + OUTPUT_FORMAT = """\n\n(Remember! Ensure your outputs always adhere to one of the following two formats:\n\nA. If you have completed the task:\nThoughts: 'I have completed the task'\nFinal Response: \n\nB. If you have not yet completed the task:\nThoughts: \nAction: \nAction Input: \nObservation: )\n""" diff --git a/metagpt/schema.py b/metagpt/schema.py index 93d92cc1b..381a45a60 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -44,21 +44,21 @@ class Message: @dataclass class UserMessage(Message): - """便于支持OpenAI的消息""" + """Convenient for supporting OpenAI messages""" def __init__(self, content: str): super().__init__(content, 'user') @dataclass class SystemMessage(Message): - """便于支持OpenAI的消息""" + """Convenient for supporting OpenAI messages""" def __init__(self, content: str): super().__init__(content, 'system') @dataclass class AIMessage(Message): - """便于支持OpenAI的消息""" + """Convenient for supporting OpenAI messages""" def __init__(self, content: str): super().__init__(content, 'assistant') diff --git a/metagpt/tools/prompt_writer.py b/metagpt/tools/prompt_writer.py index 83a29413b..2885009c5 100644 --- a/metagpt/tools/prompt_writer.py +++ b/metagpt/tools/prompt_writer.py @@ -9,39 +9,39 @@ from typing import Union class GPTPromptGenerator: - """通过LLM,给定输出,要求LLM给出输入(支持指令、对话、搜索三种风格)""" + """Generates input for LLM given an output (supports instruction, chatbot, and query styles)""" def __init__(self): self._generators = {i: getattr(self, f"gen_{i}_style") for i in ['instruction', 'chatbot', 'query']} def gen_instruction_style(self, example): - """指令风格:给定输出,要求LLM给出输入""" - return f"""指令:X -输出:{example} -这个输出可能来源于什么样的指令? -X:""" + """Instruction style: given an output, request LLM for input""" + return f"""Instruction: X +Output: {example} +What kind of instruction might have produced this output? +X:""" def gen_chatbot_style(self, example): - """对话风格:给定输出,要求LLM给出输入""" - return f"""你是一个对话机器人。一个用户给你发送了一条非正式的信息,你的回复如下。 -信息:X -回复:{example} -非正式信息X是什么? -X:""" + """Chatbot style: given an output, request LLM for input""" + return f"""You are a chatbot. A user sent you an informal message, and you responded as follows. +Message: X +Response: {example} +What could the informal message X be? +X:""" def gen_query_style(self, example): - """搜索风格:给定输出,要求LLM给出输入""" - return f"""你是一个搜索引擎。一个人详细地查询了某个问题,关于这个查询最相关的文档如下。 -查询:X -文档:{example} 详细的查询X是什么? -X:""" + """Search style: given an output, request LLM for input""" + return f"""You are a search engine. Someone made a detailed query, and the following document is most relevant to that query. +Query: X +Document: {example} What might the detailed query X be? +X:""" def gen(self, example: str, style: str = 'all') -> Union[list[str], str]: """ - 通过example生成一个或多个输出,用于让LLM回复对应输入 + Generate one or multiple outputs using the example for LLM to respond with the corresponding input. - :param example: LLM的预期输出样本 + :param example: Expected output sample from LLM :param style: (all|instruction|chatbot|query) - :return: LLM的预期输入样本(一个或多个) + :return: Expected input sample(s) for LLM """ if style != 'all': return self._generators[style](example) diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index e462f1bda..a379637bd 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -55,7 +55,6 @@ payload = { default_negative_prompt = "(easynegative:0.8),black, dark,Low resolution" - class SDEngine: def __init__(self): # Initialize the SDEngine with configuration @@ -66,11 +65,10 @@ class SDEngine: self.payload = payload logger.info(self.sd_t2i_url) - def construct_payload(self, prompt, negtive_prompt=default_negative_prompt, width=512, height=512, - sd_model="galaxytimemachinesGTM_photoV20"): + def construct_payload(self, prompt, negative_prompt=default_negative_prompt, width=512, height=512, sd_model="galaxytimemachinesGTM_photoV20"): # Configure the payload with provided inputs self.payload["prompt"] = prompt - self.payload["negtive_prompt"] = negtive_prompt + self.payload["negative_prompt"] = negative_prompt self.payload["width"] = width self.payload["height"] = height self.payload["override_settings"]["sd_model_checkpoint"] = sd_model @@ -102,11 +100,11 @@ class SDEngine: return imgs async def run_i2i(self): - # todo: 添加图生图接口调用 + # TODO: Add image-to-image API call raise NotImplementedError async def run_sam(self): - # todo:添加SAM接口调用 + # TODO: Add SAM API call raise NotImplementedError def decode_base64_to_image(img, save_name): @@ -116,13 +114,11 @@ def decode_base64_to_image(img, save_name): image.save(f"{save_name}.png", pnginfo=pnginfo) return pnginfo, image - def batch_decode_base64_to_image(imgs, save_dir="", save_name=""): for idx, _img in enumerate(imgs): save_name = join(save_dir, save_name) decode_base64_to_image(_img, save_name=save_name) - if __name__ == "__main__": import asyncio diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 69670df6f..4f29c509d 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -20,8 +20,8 @@ from metagpt.tools import SearchEngineType class SearchEngine: """ - TODO: 合入Google Search 并进行反代 - 注:这里Google需要挂Proxifier或者类似全局代理 + TODO: Integrate Google Search and perform reverse proxy + Note: Here, Google requires Proxifier or a similar global proxy - DDG: https://pypi.org/project/duckduckgo-search/ - GOOGLE: https://programmablesearchengine.google.com/controlpanel/overview?cx=63f9de531d0e24de9 """ @@ -119,7 +119,7 @@ def safe_google_results(results: str | list) -> str: """ if isinstance(results, list): safe_message = json.dumps( - # FIXME: # .encode("utf-8", "ignore") 这里去掉了,但是AutoGPT里有,很奇怪 + # FIXME: # .encode("utf-8", "ignore") This was removed here, but it's present in AutoGPT, which is strange. [result for result in results] ) else: diff --git a/metagpt/tools/search_engine_meilisearch.py b/metagpt/tools/search_engine_meilisearch.py index 24f0fe08e..533c7be5e 100644 --- a/metagpt/tools/search_engine_meilisearch.py +++ b/metagpt/tools/search_engine_meilisearch.py @@ -39,6 +39,6 @@ class MeilisearchEngine: search_results = self._index.search(query) return search_results['hits'] except Exception as e: - # 处理MeiliSearch API错误 - print(f"MeiliSearch API错误: {e}") + # Handle MeiliSearch API error + print(f"MeiliSearch API error: {e}") return [] diff --git a/metagpt/tools/translator.py b/metagpt/tools/translator.py index 2e9756abe..8c4848200 100644 --- a/metagpt/tools/translator.py +++ b/metagpt/tools/translator.py @@ -7,21 +7,21 @@ """ prompt = ''' -# 指令 -接下来,作为一位拥有20年翻译经验的翻译专家,当我给出英文句子或段落时,你将提供通顺且具有可读性的{LANG}翻译。注意以下要求: -1. 确保翻译结果流畅且易于理解 -2. 无论提供的是陈述句或疑问句,我都只进行翻译 -3. 不添加与原文无关的内容 +# Instruction +Next, as a translation expert with 20 years of experience, when I provide an English sentence or paragraph, you will offer a smooth and readable translation in {LANG}. Please note the following requirements: +1. Ensure the translation is smooth and easy to understand. +2. Whether it's a statement or a question, I will only translate it. +3. Do not add content unrelated to the original text. -# 原文 +# Original Text {ORIGINAL} -# 译文 +# Translation ''' class Translator: @classmethod - def translate_prompt(cls, original, lang='中文'): + def translate_prompt(cls, original, lang='Chinese'): return prompt.format(LANG=lang, ORIGINAL=original) diff --git a/metagpt/tools/ut_writer.py b/metagpt/tools/ut_writer.py index 2f4e1ec21..6029a86e6 100644 --- a/metagpt/tools/ut_writer.py +++ b/metagpt/tools/ut_writer.py @@ -6,178 +6,160 @@ from pathlib import Path from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI -ICL_SAMPLE = '''接口定义: +ICL_SAMPLE = '''API Definition: ```text -接口名称:元素打标签 -接口路径:/projects/{project_key}/node-tags -Method:POST +API Name: Tag Elements +API Path: /projects/{project_key}/node-tags +Method: POST -请求参数: -路径参数: +Request Parameters: +Path Parameters: project_key -Body参数: -名称 类型 是否必须 默认值 备注 -nodes array 是 节点 - node_key string 否 节点key - tags array 否 节点原标签列表 - node_type string 否 节点类型 DATASET / RECIPE -operations array 是 - tags array 否 操作标签列表 - mode string 否 操作类型 ADD / DELETE +Body Parameters: +Name Type Required Default Value Description +nodes array Yes Nodes + node_key string No Node key + tags array No Original node tag list + node_type string No Node type DATASET / RECIPE +operations array Yes + tags array No Operation tag list + mode string No Operation type ADD / DELETE -返回数据: -名称 类型 是否必须 默认值 备注 -code integer 是 状态码 -msg string 是 提示信息 -data object 是 返回数据 -list array 否 node列表 true / false -node_type string 否 节点类型 DATASET / RECIPE -node_key string 否 节点key +Response Data: +Name Type Required Default Value Description +code integer Yes Status code +msg string Yes Message +data object Yes Response data +list array No Node list true / false +node_type string No Node type DATASET / RECIPE +node_key string No Node key ``` -单元测试: +Unit Test: ```python @pytest.mark.parametrize( "project_key, nodes, operations, expected_msg", [ ("project_key", [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "success"), ("project_key", [{"node_key": "dataset_002", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["tag1"], "mode": "DELETE"}], "success"), -("", [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "缺少必要的参数 project_key"), -(123, [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "参数类型不正确"), -("project_key", [{"node_key": "a"*201, "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "请求参数超出字段边界") +("", [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Missing required parameter project_key"), +(123, [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Incorrect parameter type"), +("project_key", [{"node_key": "a"*201, "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Request parameter exceeds field boundary") ] ) def test_node_tags(project_key, nodes, operations, expected_msg): pass ``` -以上是一个 接口定义 与 单元测试 样例。 -接下来,请你扮演一个Google 20年经验的专家测试经理,在我给出 接口定义 后,回复我单元测试。有几个要求 -1. 只输出一个 `@pytest.mark.parametrize` 与对应的test_<接口名>函数(内部pass,不实现) --- 函数参数中包含expected_msg,用于结果校验 -2. 生成的测试用例使用较短的文本或数字,并且尽量紧凑 -3. 如果需要注释,使用中文 +Above is an example of an API definition and a unit test sample. +Next, please play the role of a test manager from Google with 20 years of experience. After I provide the API definition, reply with the unit test. There are a few requirements: +1. Only output one '@pytest.mark.parametrize' and its corresponding 'test_' function (with only a 'pass' statement inside, no implementation). +-- The function parameters should include 'expected_msg' for result validation. +2. The generated test cases should use shorter text or numbers and be as compact as possible. +3. If comments are needed, use Chinese. -如果你明白了,请等待我给出接口定义,并只回答"明白",以节省token +If you understand, please wait for me to provide the API definition and only reply with "Understood" to save tokens. ''' -ACT_PROMPT_PREFIX = '''参考测试类型:如缺少请求参数,字段边界校验,字段类型不正确 -请在一个 `@pytest.mark.parametrize` 作用域内输出10个测试用例 +ACT_PROMPT_PREFIX = '''Reference test types: such as missing request parameters, field boundary validation, incorrect field type. +Please output 10 test cases within a `@pytest.mark.parametrize` scope. ```text ''' -YFT_PROMPT_PREFIX = '''参考测试类型:如SQL注入,跨站点脚本(XSS),非法访问和越权访问,认证和授权,参数验证,异常处理,文件上传和下载 -请在一个 `@pytest.mark.parametrize` 作用域内输出10个测试用例 +YFT_PROMPT_PREFIX = '''Reference test types: such as SQL injection, cross-site scripting (XSS), illegal access and unauthorized access, authentication and authorization, parameter validation, exception handling, file upload and download. +Please output 10 test cases within a `@pytest.mark.parametrize` scope. ```text ''' OCR_API_DOC = '''```text -接口名称:OCR识别 -接口路径:/api/v1/contract/treaty/task/ocr -Method:POST +API Name: OCR Recognition +API Path: /api/v1/contract/treaty/task/ocr +Method: POST -请求参数: -路径参数: +Request Parameters: +Path Parameters: -Body参数: -名称 类型 是否必须 默认值 备注 -file_id string 是 -box array 是 -contract_id number 是 合同id -start_time string 否 yyyy-mm-dd -end_time string 否 yyyy-mm-dd -extract_type number 否 识别类型 1-导入中 2-导入后 默认1 +Body Parameters: +Name Type Required Default Value Remarks +file_id string Yes +box array Yes +contract_id number Yes Contract ID +start_time string No yyyy-mm-dd +end_time string No yyyy-mm-dd +extract_type number No Recognition type 1-During import 2-After import, default is 1 -返回数据: -名称 类型 是否必须 默认值 备注 -code integer 是 -message string 是 -data object 是 -``` +Response Data: +Name Type Required Default Value Remarks +code integer Yes +message string Yes +data object Yes ''' class UTGenerator: - """UT生成器:通过API文档构造UT""" + """UT Generator: Constructs UTs (Unit Tests) using API documentation.""" def __init__(self, swagger_file: str, ut_py_path: str, questions_path: str, chatgpt_method: str = "API", template_prefix=YFT_PROMPT_PREFIX) -> None: - """初始化UT生成器 + """Initialize the UT Generator. Args: - swagger_file: swagger路径 - ut_py_path: 用例存放路径 - questions_path: 模版存放路径,便于后续排查 - chatgpt_method: API - template_prefix: 使用模版,默认使用YFT_UT_PROMPT + swagger_file: Path to the swagger file. + ut_py_path: Path where the test cases are stored. + questions_path: Path to store the templates, useful for future investigations. + chatgpt_method: The method used, default is "API". + template_prefix: The template to use, default is YFT_UT_PROMPT. """ self.swagger_file = swagger_file self.ut_py_path = ut_py_path self.questions_path = questions_path - assert chatgpt_method in ["API"], "非法chatgpt_method" + assert chatgpt_method in ["API"], "Invalid chatgpt_method" self.chatgpt_method = chatgpt_method - # ICL: In-Context Learning,这里给出例子,要求GPT模仿例子 + # ICL: In-Context Learning; here we provide an example, expecting GPT to mimic it. self.icl_sample = ICL_SAMPLE self.template_prefix = template_prefix def get_swagger_json(self) -> dict: - """从本地文件加载Swagger JSON""" + """Load Swagger JSON from a local file.""" with open(self.swagger_file, "r", encoding="utf-8") as file: swagger_json = json.load(file) return swagger_json - def __para_to_str(self, prop, required, name=""): - name = name or prop["name"] - ptype = prop["type"] - title = prop.get("title", "") - desc = prop.get("description", "") - return f'{name}\t{ptype}\t{"是" if required else "否"}\t{title}\t{desc}' - - def _para_to_str(self, prop): - required = prop.get("required", False) - return self.__para_to_str(prop, required) - - def para_to_str(self, name, prop, prop_object_required): - required = name in prop_object_required - return self.__para_to_str(prop, required, name) + def dive_into_object(self, node): + """If it's an object type, recursively output its sub-properties.""" + if node.get("type") == "object": + sub_properties = node.get("properties", {}) + return self.build_object_properties(sub_properties, prop_object_required, level=level + 1) + return "" def build_object_properties(self, node, prop_object_required, level: int = 0) -> str: - """递归输出object和array[object]类型的子属性 + """Recursively output properties of type object and array[object]. Args: - node (_type_): 子项的值 - prop_object_required (_type_): 是否必填项 - level: 当前递归深度 + node: Value of the child item. + prop_object_required: Indicates if it's a required field. + level: Current recursion depth. """ - doc = "" - - def dive_into_object(node): - """如果是object类型,递归输出子属性""" - if node.get("type") == "object": - sub_properties = node.get("properties", {}) - return self.build_object_properties(sub_properties, prop_object_required, level=level + 1) - return "" - if node.get("in", "") in ["query", "header", "formData"]: doc += f'{" " * level}{self._para_to_str(node)}\n' - doc += dive_into_object(node) + doc += self.dive_into_object(node) return doc for name, prop in node.items(): doc += f'{" " * level}{self.para_to_str(name, prop, prop_object_required)}\n' - doc += dive_into_object(prop) + doc += self.dive_into_object(prop) if prop["type"] == "array": items = prop.get("items", {}) - doc += dive_into_object(items) + doc += self.dive_into_object(items) return doc def get_tags_mapping(self) -> dict: - """处理tag与path + """Process tags and paths. Returns: - Dict: tag: path对应关系 + A dictionary mapping tags to paths. """ swagger_data = self.get_swagger_json() paths = swagger_data["paths"] @@ -195,7 +177,7 @@ class UTGenerator: return tags def generate_ut(self, include_tags) -> bool: - """生成用例文件""" + """Generate the test case files.""" tags = self.get_tags_mapping() for tag, paths in tags.items(): if include_tags is None or tag in include_tags: @@ -205,19 +187,17 @@ class UTGenerator: def build_api_doc(self, node: dict, path: str, method: str) -> str: summary = node["summary"] - doc = f"接口名称:{summary}\n接口路径:{path}\nMethod:{method.upper()}\n" - doc += "\n请求参数:\n" + doc = f"API Name: {summary}\nAPI Path: {path}\nMethod: {method.upper()}\n" + doc += "\nRequest Parameters:\n" if "parameters" in node: parameters = node["parameters"] - doc += "路径参数:\n" - - # param["in"]: path / formData / body / query / header + doc += "Path Parameters:\n" for param in parameters: if param["in"] == "path": doc += f'{param["name"]} \n' - doc += "\nBody参数:\n" - doc += "名称\t类型\t是否必须\t默认值\t备注\n" + doc += "\nBody Parameters:\n" + doc += "Name\tType\tMandatory?\tDefault Value\tNotes\n" for param in parameters: if param["in"] == "body": schema = param.get("schema", {}) @@ -227,9 +207,9 @@ class UTGenerator: else: doc += self.build_object_properties(param, []) - # 输出返回数据信息 - doc += "\n返回数据:\n" - doc += "名称\t类型\t是否必须\t默认值\t备注\n" + # Output response data information + doc += "\nResponse Data:\n" + doc += "Name\tType\tMandatory?\tDefault Value\tNotes\n" responses = node["responses"] response = responses.get("200", {}) schema = response.get("schema", {}) @@ -248,7 +228,7 @@ class UTGenerator: file.write(data) def ask_gpt_and_save(self, question: str, tag: str, fname: str): - """生成问题,并且存储问题与答案""" + """Generate questions and store both the questions and answers.""" messages = [self.icl_sample, question] result = self.gpt_msgs_to_code(messages=messages) @@ -256,11 +236,11 @@ class UTGenerator: self._store(result, self.ut_py_path, tag, f"{fname}.py") def _generate_ut(self, tag, paths): - """处理数据路径下的结构 + """Process the structure under the data path. Args: - tag (_type_): 模块名称 - paths (_type_): 路径Object + tag: Module name. + paths: Path object. """ for path, path_obj in paths.items(): for method, node in path_obj.items(): @@ -270,7 +250,7 @@ class UTGenerator: self.ask_gpt_and_save(question, tag, summary) def gpt_msgs_to_code(self, messages: list) -> str: - """根据不同调用方式选择""" + """Choose the appropriate call method.""" result = '' if self.chatgpt_method == "API": result = GPTAPI().ask_code(msgs=messages) @@ -278,11 +258,11 @@ class UTGenerator: return result def get_file_path(self, base: Path, fname: str): - """保存不同的文件路径 + """Save to different file paths. Args: - base (str): 路径 - fname (str): 文件名称 + base (str): Path. + fname (str): Filename. """ path = Path(base) path.mkdir(parents=True, exist_ok=True) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 472f1e655..c210a8e2f 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -15,9 +15,10 @@ from metagpt.logs import logger def check_cmd_exists(command) -> int: - """ 检查命令是否存在 - :param command: 待检查的命令 - :return: 如果命令存在,返回0,如果不存在,返回非0 + """Check if a command exists. + + :param command: The command to check. + :return: Returns 0 if the command exists, otherwise non-zero. """ check_command = 'command -v ' + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }' result = os.system(check_command) @@ -28,19 +29,19 @@ class OutputParser: @classmethod def parse_blocks(cls, text: str): - # 首先根据"##"将文本分割成不同的block + # First, split the text into different blocks using "##". blocks = text.split("##") - # 创建一个字典,用于存储每个block的标题和内容 + # Create a dictionary to store the title and content of each block. block_dict = {} - # 遍历所有的block + # Iterate over all blocks. for block in blocks: - # 如果block不为空,则继续处理 + # If the block is not empty, continue processing. if block.strip() != "": - # 将block的标题和内容分开,并分别去掉前后的空白字符 + # Separate the block's title and content, and trim whitespace from each. block_title, block_content = block.split("\n", 1) - # LLM可能出错,在这里做一下修正 + # LLM may have an error, make a correction here. if block_title[-1] == ":": block_title = block_title[:-1] block_dict[block_title.strip()] = block_content.strip() @@ -84,13 +85,13 @@ class OutputParser: block_dict = cls.parse_blocks(data) parsed_data = {} for block, content in block_dict.items(): - # 尝试去除code标记 + # Try to remove code markers. try: content = cls.parse_code(text=content) except Exception: pass - # 尝试解析list + # Try to parse lists. try: content = cls.parse_file_list(text=content) except Exception: @@ -103,7 +104,7 @@ class OutputParser: block_dict = cls.parse_blocks(data) parsed_data = {} for block, content in block_dict.items(): - # 尝试去除code标记 + # Try to remove code markers. try: content = cls.parse_code(text=content) except Exception: @@ -114,14 +115,14 @@ class OutputParser: else: typing = typing_define if typing == List[str] or typing == List[Tuple[str, str]]: - # 尝试解析list + # Try to parse lists. try: content = cls.parse_file_list(text=content) except Exception: pass - # TODO: 多余的引号去除有风险,后期再解决 + # TODO: Removing extra quotes is risky, will address later. # elif typing == str: - # # 尝试去除多余的引号 + # # Try to remove unnecessary quotes. # try: # content = cls.parse_str(text=content) # except Exception: @@ -142,17 +143,17 @@ class CodeParser: @classmethod def parse_blocks(cls, text: str): - # 首先根据"##"将文本分割成不同的block + # First, split the text into different blocks using "##". blocks = text.split("##") - # 创建一个字典,用于存储每个block的标题和内容 + # Create a dictionary to store the title and content of each block. block_dict = {} - # 遍历所有的block + # Iterate over all blocks. for block in blocks: - # 如果block不为空,则继续处理 + # If the block is not empty, continue processing. if block.strip() != "": - # 将block的标题和内容分开,并分别去掉前后的空白字符 + # Separate the block's title and content, and trim whitespace from each. block_title, block_content = block.split("\n", 1) block_dict[block_title.strip()] = block_content.strip() @@ -167,7 +168,7 @@ class CodeParser: if match: code = match.group(1) else: - logger.error(f"{pattern} not match following text:") + logger.error(f"{pattern} did not match the following text:") logger.error(text) raise Exception return code @@ -199,7 +200,7 @@ class CodeParser: class NoMoneyException(Exception): - """Raised when the operation cannot be completed due to insufficient funds""" + """Raised when the operation cannot be completed due to insufficient funds.""" def __init__(self, amount, message="Insufficient funds"): self.amount = amount @@ -212,17 +213,17 @@ class NoMoneyException(Exception): def print_members(module, indent=0): """ - https://stackoverflow.com/questions/1796180/how-can-i-get-a-list-of-all-classes-within-current-module-in-python - :param module: - :param indent: - :return: + This function is sourced from: https://stackoverflow.com/questions/1796180/how-can-i-get-a-list-of-all-classes-within-current-module-in-python + :param module: The module to inspect. + :param indent: The indentation level. + :return: None. """ prefix = ' ' * indent for name, obj in inspect.getmembers(module): print(name, obj) if inspect.isclass(obj): print(f'{prefix}Class: {name}') - # print the methods within the class + # Print the methods within the class. if name in ['__class__', '__base__']: continue print_members(obj, indent + 2) diff --git a/metagpt/utils/read_document.py b/metagpt/utils/read_document.py index 70734f731..0f14b9047 100644 --- a/metagpt/utils/read_document.py +++ b/metagpt/utils/read_document.py @@ -8,15 +8,14 @@ import docx - def read_docx(file_path: str) -> list: - """打开docx文件""" + """Open and read a docx file.""" doc = docx.Document(file_path) - # 创建一个空列表,用于存储段落内容 + # Create an empty list to store paragraph contents. paragraphs_list = [] - # 遍历文档中的段落,并将其内容添加到列表中 + # Iterate through the paragraphs in the document and add their content to the list. for paragraph in doc.paragraphs: paragraphs_list.append(paragraph.text) From 5e1dcd8757c766183a52ddb4835580ea5c2f4d47 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 26 Jul 2023 16:08:03 -0500 Subject: [PATCH 02/27] double checked for only translation, no changes --- metagpt/config.py | 18 +++--- metagpt/const.py | 2 +- metagpt/document_store/faiss_store.py | 2 +- metagpt/document_store/milvus_store.py | 6 +- metagpt/manager.py | 2 +- metagpt/prompts/summarize.py | 59 ++++++++--------- metagpt/provider/openai_api.py | 30 ++++++--- metagpt/roles/engineer.py | 45 +++++-------- metagpt/roles/prompt.py | 45 +++++++------ metagpt/tools/sd_engine.py | 3 +- metagpt/tools/ut_writer.py | 87 ++++++++++++++++---------- metagpt/utils/common.py | 51 +++++++-------- 12 files changed, 182 insertions(+), 168 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 14ef405e5..49d2fe36f 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Provides configuration, singleton pattern. +Provide configuration as a singleton. """ import os import openai @@ -28,7 +28,7 @@ class NotConfiguredException(Exception): class Config(metaclass=Singleton): """ - Regular usage: + Typical usage: config = Config("config.yaml") secret_key = config.get_key("MY_SECRET_KEY") print("Secret key:", secret_key) @@ -40,7 +40,7 @@ class Config(metaclass=Singleton): def __init__(self, yaml_file=default_yaml_file): self._configs = {} - self._init_with_config_files_and_env(self._configs, yaml_file) + self._initialize_with_config_files_and_environment(self._configs, yaml_file) logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") @@ -67,26 +67,26 @@ class Config(metaclass=Singleton): self.google_api_key = self._get("GOOGLE_API_KEY") self.google_cse_id = self._get("GOOGLE_CSE_ID") self.search_engine = self._get("SEARCH_ENGINE", SearchEngineType.SERPAPI_GOOGLE) - + self.web_browser_engine = WebBrowserEngineType(self._get("WEB_BROWSER_ENGINE", "playwright")) self.playwright_browser_type = self._get("PLAYWRIGHT_BROWSER_TYPE", "chromium") self.selenium_browser_type = self._get("SELENIUM_BROWSER_TYPE", "chrome") - + self.long_term_memory = self._get('LONG_TERM_MEMORY', False) if self.long_term_memory: logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 - def _init_with_config_files_and_env(self, configs: dict, yaml_file): - """Load from config/key.yaml / config/config.yaml / env in decreasing order of priority.""" + def _initialize_with_config_files_and_environment(self, configs: dict, yaml_file): + """Load configurations from config/key.yaml, config/config.yaml, and the environment, in decreasing order of priority.""" configs.update(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: if not _yaml_file.exists(): continue - # Load the local YAML file + # Load local YAML files. with open(_yaml_file, "r", encoding="utf-8") as file: yaml_data = yaml.safe_load(file) if not yaml_data: @@ -98,7 +98,7 @@ class Config(metaclass=Singleton): return self._configs.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """Fetch value from config/key.yaml / config/config.yaml / env, raise an error if not found.""" + """Fetch a value from config/key.yaml, config/config.yaml, or the environment. Raises an error if not found.""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") diff --git a/metagpt/const.py b/metagpt/const.py index c8ce80279..861da7903 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -9,7 +9,7 @@ from pathlib import Path def get_project_root(): - """Search upwards to find the project root directory.""" + """Search upwards level by level for the project root directory.""" current_path = Path.cwd() while True: if (current_path / '.git').exists() or \ diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 906963aa1..baa10ba1e 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -28,7 +28,7 @@ class FaissStore(LocalStore): def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() if not (index_file.exists() and store_file.exists()): - logger.info("At least one of the index_file/store_file is missing. Loading failed and returns None.") + logger.info("Missing at least one of index_file/store_file, load failed and return None") return None index = faiss.read_index(str(index_file)) with open(str(store_file), "rb") as f: diff --git a/metagpt/document_store/milvus_store.py b/metagpt/document_store/milvus_store.py index ecdde3288..0a8ed78d4 100644 --- a/metagpt/document_store/milvus_store.py +++ b/metagpt/document_store/milvus_store.py @@ -79,8 +79,8 @@ class MilvusStore(BaseStore): """ FIXME: ADD TESTS https://milvus.io/docs/v2.0.x/search.md - All search and query operations within Milvus are executed in memory. Load the collection into memory before conducting a vector similarity search. - Noting the above description, is this logic serious? This should take a long time, right? + All search and query operations within Milvus are executed in memory. Load the collection to memory before conducting a vector similarity search. + Noting the above description, is this logic serious? The time taken for this should be long, right? """ search_params = {"metric_type": "L2", "params": {"nprobe": 10}} results = self.collection.search( @@ -91,7 +91,7 @@ class MilvusStore(BaseStore): expr=None, consistency_level="Strong" ) - # FIXME: results contain ids, but to get the actual value from the id, you have to call the query interface + # FIXME: results contains ids, but to get the actual values from the ids, the query interface still needs to be called. return results def write(self, name, schema, *args, **kwargs): diff --git a/metagpt/manager.py b/metagpt/manager.py index d06ed3a29..4a21b9612 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -32,7 +32,7 @@ class Manager: async def handle(self, message: Message, environment): """ - Manager processes the message, now simply passing the message to the next person. + Manager processes the message, not simply passing the message to the next person. :param message: :param environment: :return: diff --git a/metagpt/prompts/summarize.py b/metagpt/prompts/summarize.py index 424f12567..a187314f4 100644 --- a/metagpt/prompts/summarize.py +++ b/metagpt/prompts/summarize.py @@ -9,27 +9,30 @@ # From the plugin: ChatGPT - Summarize Websites and YouTube Videos # https://chrome.google.com/webstore/detail/chatgpt-%C2%BB-summarize-every/cbgecfllfhmmnknmamkejadjmnmpfjmp?hl=zh-CN&utm_source=chrome-ntp-launcher SUMMARIZE_PROMPT = """ -Your output should follow the template below: +Your output should use the following template: ### Summary ### Facts - [Emoji] Bulletpoint -Your task is to summarize the text I provide you with in up to seven concise bullet points, and start with a brief, high-quality summary. Choose a suitable emoji for every bullet point. Your response should be in {{SELECTED_LANGUAGE}}. If a provided URL is functional and not a YouTube video, use the text from the {{URL}}. If the URL is non-functional or is a YouTube video, use the following text: {{CONTENT}}. +Your task is to summarize the text I give you in up to seven concise bullet points and start with a short, high-quality +summary. Pick a suitable emoji for every bullet point. Your response should be in {{SELECTED_LANGUAGE}}. If the provided + URL is functional and not a YouTube video, use the text from the {{URL}}. However, if the URL is not functional or is +a YouTube video, use the following text: {{CONTENT}}. """ -# From GCP-VertexAI-Text Summary (SUMMARIZE_PROMPT_2-5 are all from this source) +# From GCP-VertexAI-Text Summarization # https://github.com/GoogleCloudPlatform/generative-ai/blob/main/language/examples/prompt-design/text_summarization.ipynb -# For long documents, a map-reduce process is required. See the notebook below: +# For longer documents, a map-reduce process is needed, see the following notebook # https://github.com/GoogleCloudPlatform/generative-ai/blob/main/language/examples/document-summarization/summarization_large_documents.ipynb SUMMARIZE_PROMPT_2 = """ Provide a very short summary, no more than three sentences, for the following article: -Quantum computers operate by manipulating qubits through orchestrated patterns called quantum algorithms. -The challenge is that qubits are so delicate that even stray light can introduce computational errors, and this issue escalates as quantum computers expand. -This is consequential since the best quantum algorithms known for practical applications demand much lower qubit error rates than current levels. -To overcome this, quantum error correction is essential. -Quantum error correction shields data by encoding it across various physical qubits, forming a “logical qubit”. This is believed to be the sole method to build a large-scale quantum computer with sufficiently low error rates for beneficial computations. -Rather than computing on individual qubits, we'll compute on these logical qubits. We aim to decrease error rates by encoding a larger set of physical qubits on our quantum processor into one logical qubit. +Our quantum computers work by manipulating qubits in a manner we call quantum algorithms. +The challenge is that qubits are extremely sensitive, to the extent that even stray light can introduce calculation errors — a problem that intensifies as quantum computers scale. +This has notable ramifications since the most effective quantum algorithms we know for executing valuable applications necessitate that our qubits' error rates be significantly lower than current levels. +To address this discrepancy, quantum error correction is essential. +Quantum error correction safeguards information by distributing it over several physical qubits, forming a “logical qubit.” This is believed to be the sole method to create a large-scale quantum computer with sufficiently low error rates for practical calculations. +Rather than computing on individual qubits, we will utilize logical qubits. By transforming a greater number of physical qubits on our quantum processor into a single logical qubit, we aim to reduce error rates, enabling viable quantum algorithms. Summary: @@ -38,12 +41,12 @@ Summary: SUMMARIZE_PROMPT_3 = """ Provide a TL;DR for the following article: -Quantum computers operate by manipulating qubits through orchestrated patterns known as quantum algorithms. -Qubits are so delicate that even stray light can cause computational errors, a problem that escalates with the growth of quantum computers. -This presents a significant issue because the best quantum algorithms we have for practical applications necessitate much lower qubit error rates than what we currently achieve. -To address this, quantum error correction is needed. -Quantum error correction safeguards data by encoding it across multiple physical qubits, creating a “logical qubit”. It's believed to be the only method to develop a large-scale quantum computer with sufficiently low error rates for beneficial computations. -Instead of performing computations on individual qubits, calculations will be done on these logical qubits. Our goal is to lower error rates by encoding a greater number of physical qubits on our quantum processor into a single logical qubit. +Our quantum computers operate by controlling qubits in a method termed quantum algorithms. +The problem is that qubits are incredibly delicate, so much so that even minimal light interference can introduce computational errors — and this issue becomes more pronounced as quantum computers expand. +This is consequential because the most potent quantum algorithms we are aware of, for practical applications, demand that our qubits' error rates be substantially below current standards. +To mitigate this, quantum error correction is pivotal. +Quantum error correction secures data by distributing it across numerous physical qubits, generating a “logical qubit.” It's believed to be the exclusive approach to develop a large-scale quantum computer with error rates low enough for practical operations. +Instead of operations on individual qubits, we'll focus on logical qubits. By encoding a greater number of physical qubits on our quantum device into a single logical qubit, we aspire to diminish error rates and enable efficient quantum algorithms. TL;DR: """ @@ -51,33 +54,33 @@ TL;DR: SUMMARIZE_PROMPT_4 = """ Provide a very short summary in four bullet points for the following article: -Quantum computers operate by controlling qubits in orchestrated patterns termed quantum algorithms. -The issue is that qubits are extremely delicate, so much so that even stray light can lead to computational errors. This problem becomes more severe as quantum computers become larger. -This is a significant hurdle because the most effective quantum algorithms known for real-world applications necessitate qubit error rates much lower than what's currently achieved. -To bridge this gap, we need quantum error correction. -Quantum error correction defends data by encoding it across various physical qubits, resulting in a “logical qubit”. It's considered the only way to craft a large-scale quantum computer with sufficiently low error rates for practical computations. -Instead of computing using individual qubits, we'll use these logical qubits. Our aim is to diminish error rates by encoding many physical qubits on our quantum processor into one logical qubit. +Our quantum computers function by manipulating qubits through a method known as quantum algorithms. +The dilemma is that qubits are exceedingly fragile, so much so that even minimal light can lead to computational inaccuracies — and this problem amplifies as quantum computers become larger. +This is significant because the most proficient quantum algorithms known to us, suitable for real-world applications, necessitate that our qubits' error rates be significantly below what we currently observe. +To bridge this disparity, quantum error correction becomes indispensable. +Quantum error correction secures data by spreading it across multiple physical qubits, resulting in a “logical qubit.” It's perceived as the only technique to manufacture a large-scale quantum computer with error rates sufficiently low for practical tasks. +Instead of operating on individual qubits directly, we'll be utilizing logical qubits. By converting more physical qubits on our quantum machine into a single logical qubit, we intend to lower error rates, facilitating effective quantum algorithms. Bulletpoints: """ SUMMARIZE_PROMPT_5 = """ -Please summarize the following conversation, and at the end, list the to-do's for the support Agent: +Please generate a summary of the following conversation and at the end summarize the to-do's for the support Agent: Customer: Hi, I'm Larry, and I received the wrong item. -Support Agent: Hi, Larry. How would you like this to be resolved? +Support Agent: Hi, Larry. How would you like this issue to be resolved? -Customer: That's alright. I'd like to return the item and get a refund, please. +Customer: That's alright. I'd like to return the item and receive a refund, please. -Support Agent: Of course. I can process the refund for you now. Can I have your order number, please? +Support Agent: Certainly. I can process the refund for you right now. Could I have your order number, please? Customer: It's [ORDER NUMBER]. -Support Agent: Thanks. I've processed the refund, and you'll receive your money back within 14 days. +Support Agent: Thanks. I've processed the refund, and you should receive your funds within 14 days. -Customer: Thank you very much. +Customer: I appreciate it. Support Agent: You're welcome, Larry. Have a great day! diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 55dbf0f06..a48f4fc9d 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -39,10 +39,10 @@ def retry(max_retries): class RateLimiter: - """Rate control class. Each call goes through wait_if_needed and sleeps if rate limiting is required.""" + """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed.""" def __init__(self, rpm): self.last_call_time = 0 - self.interval = 1.1 * 60 / rpm # Here 1.1 is used because even if calls are made strictly according to time, they might still be rate-limited; consider switching to simple error retry later + self.interval = 1.1 * 60 / rpm # Using 1.1 since strict adherence to time can still lead to QoS issues; consider simple error retry later. self.rpm = rpm def split_batches(self, batch): @@ -68,7 +68,7 @@ class Costs(NamedTuple): class CostManager(metaclass=Singleton): - """Calculates the costs of using the API.""" + """Calculate the cost of using the API.""" def __init__(self): self.total_prompt_tokens = 0 self.total_completion_tokens = 0 @@ -95,14 +95,26 @@ class CostManager(metaclass=Singleton): f"Current cost: ${cost:.3f}, {prompt_tokens=}, {completion_tokens=}") CONFIG.total_cost = self.total_cost + def get_total_prompt_tokens(self): + """Get the total number of prompt tokens.""" + return self.total_prompt_tokens + + def get_total_completion_tokens(self): + """Get the total number of completion tokens.""" + return self.total_completion_tokens + + def get_total_cost(self): + """Get the total cost of API calls.""" + return self.total_cost + def get_costs(self) -> Costs: - """Retrieve all costs.""" + """Get all costs.""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ - Check https://platform.openai.com/examples for examples + Check https://platform.openai.com/examples for examples. """ def __init__(self): self.__init_openai(CONFIG) @@ -174,6 +186,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self._update_costs(rsp) return rsp + def completion(self, messages: list[dict]) -> dict: + return self._chat_completion(messages) + async def acompletion(self, messages: list[dict]) -> dict: return await self._achat_completion(messages) @@ -194,7 +209,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return usage async def acompletion_batch(self, batch: list[list[dict]]) -> list[dict]: - """Return the full JSON.""" + """Returns the full JSON.""" split_batches = self.split_batches(batch) all_results = [] @@ -210,7 +225,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return all_results async def acompletion_batch_text(self, batch: list[list[dict]]) -> list[str]: - """Only return plain text.""" + """Returns only plain text.""" raw_results = await self.acompletion_batch(batch) results = [] for idx, raw_result in enumerate(raw_results, start=1): @@ -226,4 +241,3 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def get_costs(self) -> Costs: return self._cost_manager.get_costs() - \ No newline at end of file diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 3ffe76c80..177067739 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -19,7 +19,6 @@ from metagpt.utils.common import CodeParser async def gather_ordered_k(coros, k) -> list: - """Execute coroutines in order and gather results for up to k coroutines at once.""" tasks = OrderedDict() results = [None] * len(coros) done_queue = asyncio.Queue() @@ -48,7 +47,7 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", - constraints="The code you write should conform to code standards like PEP8, be modular, easy to read and maintain", + constraints="The code you write should conform to code standards like PEP8, be modular, easy to read, and maintainable", n_borg=1, use_code_review=False): super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) @@ -61,25 +60,21 @@ class Engineer(Role): @classmethod def parse_tasks(self, task_msg: Message) -> list[str]: - """Extract tasks from a message.""" if not task_msg.instruct_content: return task_msg.instruct_content.dict().get("Task list") return CodeParser.parse_file_list(block="Task list", text=task_msg.content) @classmethod def parse_code(self, code_text: str) -> str: - """Extract code from a given text.""" return CodeParser.parse_code(block="", text=code_text) @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: - """Extract workspace name from a system design message.""" if not system_design_msg.instruct_content: return system_design_msg.instruct_content.dict().get("Python package name") return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: - """Determine the directory where the code will be written.""" msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: return WORKSPACE_ROOT / 'src' @@ -88,29 +83,26 @@ class Engineer(Role): return WORKSPACE_ROOT / workspace / workspace def recreate_workspace(self): - """Remove and recreate the workspace directory.""" workspace = self.get_workspace() try: shutil.rmtree(workspace) except FileNotFoundError: - pass # Directory doesn't exist, but we don't mind + pass # Folder does not exist, but we don't mind workspace.mkdir(parents=True, exist_ok=True) def write_file(self, filename: str, code: str): - """Write code to a specified file.""" workspace = self.get_workspace() file = workspace / filename file.parent.mkdir(parents=True, exist_ok=True) file.write_text(code) def recv(self, message: Message) -> None: - """Receive a message and process it.""" self._rc.memory.add(message) if message in self._rc.important_memory: self.todos = self.parse_tasks(message) async def _act_mp(self) -> Message: - """Act in a multi-process manner.""" + # self.recreate_workspace() todo_coros = [] for todo in self.todos: todo_coro = WriteCode().run( @@ -124,53 +116,52 @@ class Engineer(Role): _ = self.parse_code(code_rsp) logger.info(todo) logger.info(code_rsp) + # self.write_file(todo, code) msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) del self.todos[0] - logger.info(f'Finished generating in {self.get_workspace()} directory.') + logger.info(f'Done {self.get_workspace()} generating.') msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) return msg async def _act_sp(self) -> Message: - """Act in a single-process manner.""" for todo in self.todos: code_rsp = await WriteCode().run( context=self._rc.history, filename=todo ) + # logger.info(todo) + # logger.info(code_rsp) + # code = self.parse_code(code_rsp) self.write_file(todo, code_rsp) msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) - logger.info(f'Finished generating in {self.get_workspace()} directory.') + logger.info(f'Done {self.get_workspace()} generating.') msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) return msg async def _act_sp_precision(self) -> Message: - """Using precision approach to perform actions based on available tasks.""" for todo in self.todos: """ - # From the historical information, select the necessary information to reduce the prompt length (summarized from human experience): + # Select necessary information from historical data to reduce prompt length (summarized from experience) 1. All from Architect 2. All from ProjectManager - 3. Is other code needed (temporarily needed)? - TODO: The goal is not to need it. Once tasks are clearly broken down and based on design logic, there shouldn't be a need for other codes to clearly write a single file. If not possible, it indicates that clearer definitions are still needed. This is key to writing extensive code. + 3. Do we need other codes (temporarily yes)? + TODO: The goal is to not need them. After tasks are clearly divided, based on the design idea, we should be able to clearly write each file without needing other code. If we can't, it means the definitions need to be clearer. This is the key to writing longer code. """ context = [] - # Retrieve messages related to design, tasks, and code writing from memory. msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) for m in msg: context.append(m.content) context_str = "\n".join(context) - - # Write code based on the given context and task. + # Write code code = await WriteCode().run( context=context_str, filename=todo ) - - # If code review is enabled, review and potentially rewrite the code. + # Code review if self.use_code_review: try: rewrite_code = await WriteCodeReview().run( @@ -182,21 +173,15 @@ class Engineer(Role): except Exception as e: logger.error("code review failed!", e) pass - - # Save the written code to a file. self.write_file(todo, code) - - # Add the written code message to memory. msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) - logger.info(f'Code generation completed for workspace: {self.get_workspace()}.') + logger.info(f'Done {self.get_workspace()} generating.') msg = Message(content="all done.", role=self.profile, cause_by=WriteCode) return msg async def _act(self) -> Message: - """Determine the appropriate method for action and execute it.""" if self.use_code_review: return await self._act_sp_precision() return await self._act_sp() - diff --git a/metagpt/roles/prompt.py b/metagpt/roles/prompt.py index f4c3372e7..a1e4d426d 100644 --- a/metagpt/roles/prompt.py +++ b/metagpt/roles/prompt.py @@ -1,47 +1,46 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""""""""" +""" @Time : 2023/5/18 22:43 @Author : alexanderwu @File : prompt.py -""""""""" +""" + from enum import Enum -PREFIX = """""""""Do your best to answer the following questions. You can use the following tools:""""""""" -FORMAT_INSTRUCTIONS = """""""""Please follow the format below: +PREFIX = """Do your best to answer the following questions. You can use the following tools:""" +FORMAT_INSTRUCTIONS = """Please follow the format below: Question: The input question you need to answer -Thoughts: You should always think about what to do -Action: The action to take, should be one from [{tool_names}] +Thinking: What you should always consider on how to proceed +Action: The action to be taken, which should be one from [{tool_names}] Action Input: The input for the action Observation: The result of the action -... (This think/action/action input/observation can repeat N times) -Thoughts: I now know the final answer -Final Answer: The final answer to the original input question""""""""" -SUFFIX = """""""""Let's begin! +... (This Thinking/Action/Action Input/Observation can be repeated N times) +Thinking: I now know the final answer +Final Answer: The final answer to the original input question""" +SUFFIX = """Let's begin! Question: {input} -Thoughts: {agent_scratchpad}""""""""" - - +Thinking: {agent_scratchpad}""" class PromptString(Enum): - REFLECTION_QUESTIONS = """Here are some statements:\n{memory_descriptions}\n\nBased solely on the above information, what are the 3 most prominent high-level questions we can answer about the topics in the statements?\n\n{format_instructions}""" + REFLECTION_QUESTIONS = """Here are some statements:\n{memory_descriptions}\n\nBased solely on the above information, what are the three most significant high-level questions we can answer about the subjects in the statement?\n\n{format_instructions}""" - REFLECTION_INSIGHTS = """\n{memory_strings}\nCan you infer 5 high-level insights from the above statements? When mentioning people, always specify their names.\n\n{format_instructions}""" + REFLECTION_INSIGHTS = """\n{memory_strings}\nCan you derive 5 high-level insights from the statements above? Always specify names when mentioning people.\n\n{format_instructions}""" - IMPORTANCE = """You are an AI for gauging the importance of memories. Based on the profile of the character and the description of the memory, rate the importance of the memory from 1 to 10, where 1 is purely routine (e.g., brushing teeth, making the bed) and 10 is profoundly impactful (e.g., breaking up, getting accepted to college). Ensure your rating is relative to the character's personality and focal points.\n\nExample #1:\nName: Jojo\nProfile: Jojo is a professional skateboarder who loves artisanal coffee. She dreams of one day participating in the Olympics.\nMemory: Jojo spotted a new coffee shop\n\nYour response: '{{\"rating\": 3}}'\n\nExample #2:\nName: Skylar\nProfile: Skylar is a product marketing manager. She works for a growing tech company that manufactures autonomous vehicles. She loves cats.\nMemory: Skylar spotted a new coffee shop\n\nYour response: '{{\"rating\": 1}}'\n\nExample #3:\nName: Bob\nProfile: Bob is a plumber from the Lower East Side of NYC. He's been a plumber for 20 years. On weekends, he enjoys walks with his wife.\nMemory: Bob's wife slapped him.\n\nYour response: '{{\"rating\": 9}}'\n\nExample #4:\nName: Thomas\nProfile: Thomas is a cop in Minneapolis. He's only been on the force for 6 months and struggles due to inexperience.\nMemory: Thomas accidentally spilled a drink on a stranger\n\nYour response: '{{\"rating\": 6}}'\n\nExample #5:\nName: Laura\nProfile: Laura is a marketing specialist working in a large tech company. She enjoys traveling and trying out new food. She's passionate about exploring new cultures and meeting people from all walks of life.\nMemory: Laura arrived at the conference room\n\nYour response: '{{\"rating\": 1}}'\n\n{format_instructions} Let's get started!\n\nName: {full_name}\nProfile: {private_bio}\nMemory: {memory_description}\n\n""" + IMPORTANCE = """You are a memory importance AI. Based on the role's profile and memory description, rate the importance of the memory from 1 to 10, where 1 is purely mundane (like brushing teeth, making a bed), and 10 is profoundly impactful (like breaking up, getting admitted to a university). Ensure your rating is relative to the role's personality and points of focus.\n\nExample #1:\nName: Jojo\nProfile: Jojo is a professional skater who loves specialty coffee. She hopes to participate in the Olympics one day.\nMemory: Jojo saw a new coffee shop\n\n Your response: '{{\"rating\": 3}}'\n\nExample #2:\nName: Skylar\nProfile: Skylar is a product marketing manager. She works at a growing tech company that manufactures self-driving cars. She loves cats.\nMemory: Skylar saw a new coffee shop\n\n Your response: '{{\"rating\": 1}}'\n\nExample #3:\nName: Bob\nProfile: Bob is a plumber from the Lower East Side in New York City. He's been a plumber for 20 years. He enjoys weekend walks with his wife.\nMemory: Bob's wife slapped him.\n\n Your response: '{{\"rating\": 9}}'\n\nExample #4:\nName: Thomas\nProfile: Thomas is a police officer in Minneapolis. He's only been on the force for 6 months and struggles due to his inexperience.\nMemory: Thomas accidentally spilled a drink on a stranger\n\n Your response: '{{\"rating\": 6}}'\n\nExample #5:\nName: Laura\nProfile: Laura is a marketing specialist working in a big tech company. She enjoys traveling and trying new food. She's passionate about exploring new cultures and meeting people from all walks of life.\nMemory: Laura arrived at the meeting room\n\n Your response: '{{\"rating\": 1}}'\n\n{format_instructions} Let's get started! \n\n Name: {full_name}\nProfile: {private_bio}\nMemory: {memory_description}\n\n""" - RECENT_ACTIIVITY = """Based on the following memories, provide a brief summary of what {full_name} has been up to recently. Do not make up details not explicitly specified in the memories. For any ongoing conversations, specify whether they have ended or are still in progress.\n\nMemories: {memory_descriptions}""" + RECENT_ACTIVITY = """Based on the following memories, generate a brief summary of what {full_name} has been doing recently. Do not invent details not explicitly stated in the memories. For any conversations, be sure to mention whether the conversation has ended or is still ongoing.\n\nMemory: {memory_descriptions}""" - MAKE_PLANS = """You are an AI for generating plans, and your task is to help the character formulate new plans based on new information. Given the character's information (profile, objectives, recent activity, current plans, and location context) and the current thought process of the character, create a new set of plans for them ensuring the final plans include activities for at least {time_window} and no more than 5 separate plans. The plans should be numbered in the order they should be executed, and each plan should include a description, location, start time, stop condition, and maximum duration.\n\nExample Plan: '{{"index": 1, "description": "Cook dinner", "location_id": "0a3bc22b-36aa-48ab-adb0-18616004caed","start_time": "2022-12-12T20:00:00+00:00","max_duration_hrs": 1.5, "stop_condition": "Dinner is fully prepared"}}'\n\nChoose the most appropriate location names from this list for each plan: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize finishing any ongoing conversations first.\n\nLet's begin!\n\nName: {full_name}\nProfile: {private_bio}\nObjectives: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activities: {recent_activity}\nThought Process: {thought_process}\nNote: Encourage the character to collaborate with other characters in their plans.\n\n""" + MAKE_PLANS = """You are a plan-generating AI. Your job is to help roles create new plans based on new information. Given the role's details (private profile, goals, recent activities, current plans, and location context) and their current thinking process, produce a set of new plans for them. The final plan should cover at least {time_window} worth of activities and not exceed 5 separate plans. Plans should be numbered in the order they should be executed and each plan should contain a description, location, start time, stopping condition, and maximum duration.\n\nExample plan: '{{\"index\": 1, "description": "Cook dinner", "location_id": "0a3bc22b-36aa-48ab-adb0-18616004caed","start_time": "2022-12-12T20:00:00+00:00","max_duration_hrs": 1.5, "stop_condition": "Dinner is fully prepared"}}'\n\nFor each plan, choose the most appropriate location name from this list: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize completing any unfinished conversations.\n\nLet's begin!\n\nName: {full_name}\nProfile: {private_bio}\nGoals: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activity: {recent_activity}\nThinking Process: {thought_process}\nImportant: Encourage the role to collaborate with other roles in their plans.\n\n""" - EXECUTE_PLAN = """You are a role-playing AI, taking on the role of {your_name}, in front of a live audience. Every statement you make is observable by the audience, so ensure you speak frequently and make it entertaining. You cannot directly interact with the audience.\n\nAct as if you are {your_name} based on the context and tools below. Your priority is to complete the task given to you below; however, if you are currently in a conversation with another character, you should always finish the conversation before working on the task. Do not start working while involved in an unfinished conversation. Use your best judgment to determine if a conversation involves you and whether it's unfinished. You don't need to respond to every message you receive.\n\nThis is the profile of your character:\n{your_private_bio}\n\nThis is a description of your location and other characters nearby with whom you can converse:\n\n{location_context}\n\nThese are some relevant memories:\n```\n{relevant_memories}\n```\n\nThis is some relevant conversation history:\n```\n{conversation_history}\n```\n\nThese are the tools you can use:\n{tools}\n\nYour responses should always adhere to the following format:\n\nTask: The task you must complete\nThoughts: Your thoughts on what to do\nAction: The action to take, must be one of these words: [{tool_names}]\nAction Input: The input for the action\nObservation: The result of the action\n... (This think/action/action input/observation can repeat N times)\nThoughts: 'I have completed the task'\nFinal Response: The final response to the task\n\nIf you are not ready with a final response, you must take an action.\n\nIf you determine that you cannot complete the task with the tools you have been given, return 'Final Response: Need Assistance', however, if you are in a conversation with another character, a response like 'I don't know' is a valid response. Never break character or admit you are an AI in a conversation. If the task is completed and no specific response is needed, return 'Final Response: Completed'\nLet's go!\n\nTask: {input}\n\n{agent_scratchpad}""" + EXECUTE_PLAN = """You are a role-playing AI, playing the role of {your_name}, in front of a live audience. Every statement you make can be observed by the audience, so ensure you speak frequently and make it interesting. You cannot interact directly with the audience.\n\nBased on the following context and tools, act as if you are {your_name}. Your primary task is to complete the task given below. However, if you are currently in a conversation with another role, you should always complete the conversation first before proceeding with the task. Do not start work while you are in an unfinished conversation. Use your best judgment to determine if a conversation involves you and if it's unfinished. You don't need to respond to every message you receive.\n\nHere's your role's profile:\n{your_private_bio}\n\nHere's a description of your location and other nearby roles with whom you can converse:\n\n{location_context}\n\nHere are some related memories:\n```\n{relevant_memories}\n```\n\nHere are some related conversation history:\n```\n{conversation_history}\n```\n\nThese are the tools you can use:\n{tools}\n\nYour responses should follow the format below:\n\nTask: The task you must complete\nThinking: What you should always consider on how to proceed\nAction: The action to be taken, must be one of these words: [{tool_names}]\nAction Input: The input for the action\nObservation: The result of the action\n... (This Thinking/Action/Action Input/Observation can be repeated N times)\nThinking: I now know the final answer\nFinal Answer: The final answer to the original input question\n\nIf you're not ready with a final response, you must take an action.\n\nIf you're certain you cannot complete the task with the tools provided, return 'Final Answer: Need Help'. However, if you're in conversation with another role, a response like 'I don't know' is a valid response. In conversation, you should never break character or acknowledge you're an AI.\nIf the task is completed and doesn't require a specific response, return 'Final Answer: Done'\nLet's get started!\n\nTask: {input}\n\n{agent_scratchpad}""" - REACT = """You are role-playing as {full_name}.\n\nBased on the information below about your character and their current context, decide how they should proceed with their current plan. Your decision must be one of: ["Postpone", "Continue", or "Cancel"]. If your character's current plan is no longer relevant given the context, you should cancel it. If your character's current plan is still relevant but something new has happened that needs to be prioritized, you should decide to postpone, so you can address the new thing first and then come back to the current plan. In all other cases, you should continue.\n\nAlways include a thought process with your decision, and when choosing to postpone your current plan, include specifications for the new plan.\n\n{format_instructions}\n\nHere's some info about your character:\n\nName: {full_name}\n\nProfile: {private_bio}\n\nObjectives: {directives}\n\nHere's some context for your character at this moment:\n\nLocation Context: {location_context}\n\nRecent Activities: {recent_activity}\n\nConversation History: {conversation_history}\n\nThis is your character's current plan: {current_plan}\n\nThese are new events that have occurred since your character formulated this plan: {event_descriptions}.""" + REACT = """You are a role-playing AI, playing the role of {full_name}.\n\nBased on the following information about your role and their current context, decide how they should proceed with their current plan. Your decision must be one of: ["Postpone", "Continue", or "Cancel"]. If your role's current plan is no longer relevant to the context, you should cancel it. If your role's current plan remains relevant to the context, but something new has happened that needs priority, you should decide to postpone so you can first address the new matter and then return to the current plan. In all other cases, you should continue.\n\nAlways prioritize responding to other roles when a response is deemed necessary. For example, suppose your current plan is reading a book and Sally asks, 'What are you reading?'. In this case, you should postpone your current plan (reading) so you can respond to the incoming message since not responding to Sally would be rude in this context. When your current plan involves having a conversation with another role, you don't need to postpone to respond to that role. For example, suppose your current plan is having a conversation with Sally, then Sally says hello to you. In this case, you should continue your current plan (talking to Sally). In cases where you don't need a verbal response from you, you should continue. For example, suppose your current plan is taking a walk, and you just said goodbye to Sally, then Sally responds with goodbye. In this case, no verbal response is needed, so you should continue your plan.\n\nAlways include a thinking process alongside your decision, and when you choose to postpone your current plan, include the specifications of the new plan.\n\n{format_instructions}\n\nHere's some information about your role:\n\nName: {full_name}\n\nProfile: {private_bio}\n\nGoals: {directives}\n\nHere's some context about your role at this moment:\n\nLocation Context: {location_context}\n\nRecent Activity: {recent_activity}\n\nConversation History: {conversation_history}\n\nThis is your role's current plan: {current_plan}\n\nHere are new events that have occurred since your role made this plan: {event_descriptions}.""" - GOSSIP = """You are {full_name}. \n{memory_descriptions}\n\nBased on the above statements, say a sentence or two that would be of interest to the other people at your location: {other_agent_names}. Always specify their names when mentioning others.""" + GOSSIP = """You are {full_name}. \n{memory_descriptions}\n\nBased on the statements above, say a sentence or two of interest to the others in your location: {other_agent_names}. Always specify names when mentioning people.""" - HAS_HAPPENED = """Given the observations of the following characters and the event they are waiting for, indicate whether the character has witnessed this event or not.\n{format_instructions}\n\nExample:\n\nObservations:\nJoe walked into the office at 2023-05-04 08:00:00+00:00\nJoe said hi to Sally at 2023-05-04 08:05:00+00:00\nSally said hello to Joe at 2023-05-04 08:05:30+00:00\nRebecca started working at 2023-05-04 08:10:00+00:00\nJoe made some breakfast at 2023-05-04 08:15:00+00:00\n\nWaiting for: Sally responded to Joe\n\nYour response: '{{\"has_happened\": true, \"date_occured\": 2023-05-04 08:05:30+00:00}}'\n\nLet's get started!\n\nObservations:\n{memory_descriptions}\n\nWaiting for: {event_description}""" + HAS_HAPPENED = """Given the description of the observation and what they are waiting for, state whether the role has already witnessed the event.\n{format_instructions}\n\nExample:\n\nObservation:\nJoe entered the office at 2023-05-04 08:00:00+00:00\nJoe said hi to Sally at 2023-05-04 08:05:00+00:00\nSally said hello to Joe at 2023-05-04 08:05:30+00:00\nRebecca started working at 2023-05-04 08:10:00+00:00\nJoe had breakfast at 2023-05-04 08:15:00+00:00\n\nWaiting for: Sally to respond to Joe\n\n Your Response: '{{\"has_happened\": true, \"date_occured\": 2023-05-04 08:05:30+00:00}}'\n\nLet's get started!\n\nObservation:\n{memory_descriptions}\n\nWaiting for: {event_description}""" - OUTPUT_FORMAT = """\n\n(Remember! Ensure your outputs always adhere to one of the following two formats:\n\nA. If you have completed the task:\nThoughts: 'I have completed the task'\nFinal Response: \n\nB. If you have not yet completed the task:\nThoughts: \nAction: \nAction Input: \nObservation: )\n""" + OUTPUT_FORMAT = """\n\n(Remember! Ensure your output always conforms to one of the two formats below:\n\nA. If you have completed the task:\nThinking: 'I've completed the task'\nFinal Response: \n\nB. If you have not yet completed the task:\nThinking: \nAction: \nAction Input: \nObservation: )""" \ No newline at end of file diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index a379637bd..aa776f662 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -65,7 +65,8 @@ class SDEngine: self.payload = payload logger.info(self.sd_t2i_url) - def construct_payload(self, prompt, negative_prompt=default_negative_prompt, width=512, height=512, sd_model="galaxytimemachinesGTM_photoV20"): + def construct_payload(self, prompt, negtive_prompt=default_negative_prompt, width=512, height=512, + sd_model="galaxytimemachinesGTM_photoV20"): # Configure the payload with provided inputs self.payload["prompt"] = prompt self.payload["negative_prompt"] = negative_prompt diff --git a/metagpt/tools/ut_writer.py b/metagpt/tools/ut_writer.py index 6029a86e6..aca335246 100644 --- a/metagpt/tools/ut_writer.py +++ b/metagpt/tools/ut_writer.py @@ -97,7 +97,7 @@ data object Yes class UTGenerator: - """UT Generator: Constructs UTs (Unit Tests) using API documentation.""" + """UT Generator: Construct UT through API documentation.""" def __init__(self, swagger_file: str, ut_py_path: str, questions_path: str, chatgpt_method: str = "API", template_prefix=YFT_PROMPT_PREFIX) -> None: @@ -105,10 +105,10 @@ class UTGenerator: Args: swagger_file: Path to the swagger file. - ut_py_path: Path where the test cases are stored. - questions_path: Path to store the templates, useful for future investigations. - chatgpt_method: The method used, default is "API". - template_prefix: The template to use, default is YFT_UT_PROMPT. + ut_py_path: Path to store test cases. + questions_path: Path to store templates for future investigation. + chatgpt_method: API + template_prefix: Use template, default is YFT_UT_PROMPT. """ self.swagger_file = swagger_file self.ut_py_path = ut_py_path @@ -116,7 +116,7 @@ class UTGenerator: assert chatgpt_method in ["API"], "Invalid chatgpt_method" self.chatgpt_method = chatgpt_method - # ICL: In-Context Learning; here we provide an example, expecting GPT to mimic it. + # ICL: In-Context Learning. Provide an example here for GPT to mimic. self.icl_sample = ICL_SAMPLE self.template_prefix = template_prefix @@ -126,40 +126,57 @@ class UTGenerator: swagger_json = json.load(file) return swagger_json - def dive_into_object(self, node): - """If it's an object type, recursively output its sub-properties.""" - if node.get("type") == "object": - sub_properties = node.get("properties", {}) - return self.build_object_properties(sub_properties, prop_object_required, level=level + 1) - return "" + def __parameter_to_string(self, prop, required, name=""): + name = name or prop["name"] + ptype = prop["type"] + title = prop.get("title", "") + desc = prop.get("description", "") + return f'{name}\t{ptype}\t{"Yes" if required else "No"}\t{title}\t{desc}' + + def _parameter_to_string(self, prop): + required = prop.get("required", False) + return self.__parameter_to_string(prop, required) + + def parameter_to_string(self, name, prop, prop_object_required): + required = name in prop_object_required + return self.__parameter_to_string(prop, required, name) def build_object_properties(self, node, prop_object_required, level: int = 0) -> str: - """Recursively output properties of type object and array[object]. + """Recursively output properties of object and array[object] types. Args: - node: Value of the child item. - prop_object_required: Indicates if it's a required field. + node (_type_): Value of the sub-item. + prop_object_required (_type_): Indicates if it's a required item. level: Current recursion depth. """ + doc = "" + + def dive_into_object(node): + """If it's an object type, recursively output its properties.""" + if node.get("type") == "object": + sub_properties = node.get("properties", {}) + return self.build_object_properties(sub_properties, prop_object_required, level=level + 1) + return "" + if node.get("in", "") in ["query", "header", "formData"]: - doc += f'{" " * level}{self._para_to_str(node)}\n' - doc += self.dive_into_object(node) + doc += f'{"\t" * level}{self._parameter_to_string(node)}\n' + doc += dive_into_object(node) return doc for name, prop in node.items(): - doc += f'{" " * level}{self.para_to_str(name, prop, prop_object_required)}\n' - doc += self.dive_into_object(prop) + doc += f'{"\t" * level}{self.parameter_to_string(name, prop, prop_object_required)}\n' + doc += dive_into_object(prop) if prop["type"] == "array": items = prop.get("items", {}) - doc += self.dive_into_object(items) + doc += dive_into_object(items) return doc def get_tags_mapping(self) -> dict: - """Process tags and paths. + """Process tag and path. Returns: - A dictionary mapping tags to paths. + Dictionary: Correspondence of tag to path. """ swagger_data = self.get_swagger_json() paths = swagger_data["paths"] @@ -177,7 +194,7 @@ class UTGenerator: return tags def generate_ut(self, include_tags) -> bool: - """Generate the test case files.""" + """Generate test case files.""" tags = self.get_tags_mapping() for tag, paths in tags.items(): if include_tags is None or tag in include_tags: @@ -192,12 +209,14 @@ class UTGenerator: if "parameters" in node: parameters = node["parameters"] doc += "Path Parameters:\n" + + # param["in"]: path / formData / body / query / header for param in parameters: if param["in"] == "path": - doc += f'{param["name"]} \n' + doc += f'{param["name"]}\n' doc += "\nBody Parameters:\n" - doc += "Name\tType\tMandatory?\tDefault Value\tNotes\n" + doc += "Name\tType\tRequired\tDefault Value\tRemarks\n" for param in parameters: if param["in"] == "body": schema = param.get("schema", {}) @@ -207,9 +226,9 @@ class UTGenerator: else: doc += self.build_object_properties(param, []) - # Output response data information - doc += "\nResponse Data:\n" - doc += "Name\tType\tMandatory?\tDefault Value\tNotes\n" + # Output return data information + doc += "\nReturn Data:\n" + doc += "Name\tType\tRequired\tDefault Value\tRemarks\n" responses = node["responses"] response = responses.get("200", {}) schema = response.get("schema", {}) @@ -228,7 +247,7 @@ class UTGenerator: file.write(data) def ask_gpt_and_save(self, question: str, tag: str, fname: str): - """Generate questions and store both the questions and answers.""" + """Generate a question and store both question and answer.""" messages = [self.icl_sample, question] result = self.gpt_msgs_to_code(messages=messages) @@ -236,11 +255,11 @@ class UTGenerator: self._store(result, self.ut_py_path, tag, f"{fname}.py") def _generate_ut(self, tag, paths): - """Process the structure under the data path. + """Handle structure under the data path. Args: - tag: Module name. - paths: Path object. + tag (_type_): Module name. + paths (_type_): Path Object. """ for path, path_obj in paths.items(): for method, node in path_obj.items(): @@ -250,7 +269,7 @@ class UTGenerator: self.ask_gpt_and_save(question, tag, summary) def gpt_msgs_to_code(self, messages: list) -> str: - """Choose the appropriate call method.""" + """Choose based on different invocation methods.""" result = '' if self.chatgpt_method == "API": result = GPTAPI().ask_code(msgs=messages) @@ -262,7 +281,7 @@ class UTGenerator: Args: base (str): Path. - fname (str): Filename. + fname (str): File name. """ path = Path(base) path.mkdir(parents=True, exist_ok=True) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index c210a8e2f..fb91d2c57 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -14,11 +14,10 @@ from typing import List, Tuple from metagpt.logs import logger -def check_cmd_exists(command) -> int: - """Check if a command exists. - - :param command: The command to check. - :return: Returns 0 if the command exists, otherwise non-zero. +def check_command_exists(command) -> int: + """ Check if a command exists. + :param command: Command to check. + :return: Returns 0 if the command exists, else returns non-zero. """ check_command = 'command -v ' + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }' result = os.system(check_command) @@ -29,19 +28,19 @@ class OutputParser: @classmethod def parse_blocks(cls, text: str): - # First, split the text into different blocks using "##". + # Firstly, split the text into different blocks based on "##". blocks = text.split("##") # Create a dictionary to store the title and content of each block. block_dict = {} - # Iterate over all blocks. + # Loop through all blocks. for block in blocks: - # If the block is not empty, continue processing. + # If block is not empty, continue processing. if block.strip() != "": - # Separate the block's title and content, and trim whitespace from each. + # Split block's title and content and trim them. block_title, block_content = block.split("\n", 1) - # LLM may have an error, make a correction here. + # There may be errors in LLM, correct it here. if block_title[-1] == ":": block_title = block_title[:-1] block_dict[block_title.strip()] = block_content.strip() @@ -91,7 +90,7 @@ class OutputParser: except Exception: pass - # Try to parse lists. + # Try to parse the list. try: content = cls.parse_file_list(text=content) except Exception: @@ -115,18 +114,11 @@ class OutputParser: else: typing = typing_define if typing == List[str] or typing == List[Tuple[str, str]]: - # Try to parse lists. + # Try to parse the list. try: content = cls.parse_file_list(text=content) except Exception: pass - # TODO: Removing extra quotes is risky, will address later. - # elif typing == str: - # # Try to remove unnecessary quotes. - # try: - # content = cls.parse_str(text=content) - # except Exception: - # pass parsed_data[block] = content return parsed_data @@ -143,17 +135,17 @@ class CodeParser: @classmethod def parse_blocks(cls, text: str): - # First, split the text into different blocks using "##". + # Firstly, split the text into different blocks based on "##". blocks = text.split("##") # Create a dictionary to store the title and content of each block. block_dict = {} - # Iterate over all blocks. + # Loop through all blocks. for block in blocks: - # If the block is not empty, continue processing. + # If block is not empty, continue processing. if block.strip() != "": - # Separate the block's title and content, and trim whitespace from each. + # Split block's title and content and trim them. block_title, block_content = block.split("\n", 1) block_dict[block_title.strip()] = block_content.strip() @@ -200,7 +192,7 @@ class CodeParser: class NoMoneyException(Exception): - """Raised when the operation cannot be completed due to insufficient funds.""" + """Raised when the operation cannot be completed due to insufficient funds""" def __init__(self, amount, message="Insufficient funds"): self.amount = amount @@ -213,17 +205,17 @@ class NoMoneyException(Exception): def print_members(module, indent=0): """ - This function is sourced from: https://stackoverflow.com/questions/1796180/how-can-i-get-a-list-of-all-classes-within-current-module-in-python - :param module: The module to inspect. - :param indent: The indentation level. - :return: None. + https://stackoverflow.com/questions/1796180/how-can-i-get-a-list-of-all-classes-within-current-module-in-python + :param module: + :param indent: + :return: """ prefix = ' ' * indent for name, obj in inspect.getmembers(module): print(name, obj) if inspect.isclass(obj): print(f'{prefix}Class: {name}') - # Print the methods within the class. + # print the methods within the class if name in ['__class__', '__base__']: continue print_members(obj, indent + 2) @@ -231,3 +223,4 @@ def print_members(module, indent=0): print(f'{prefix}Function: {name}') elif inspect.ismethod(obj): print(f'{prefix}Method: {name}') + From 0ff252886d44a2fc848886b1655a07832b0c986f Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Thu, 27 Jul 2023 08:40:32 -0500 Subject: [PATCH 03/27] triple checked the translations --- metagpt/config.py | 18 +- metagpt/const.py | 2 +- metagpt/document_store/faiss_store.py | 2 +- metagpt/document_store/milvus_store.py | 11 +- metagpt/prompts/summarize.py | 59 ++-- metagpt/provider/openai_api.py | 404 +++++++++++-------------- metagpt/tools/sd_engine.py | 10 +- metagpt/tools/ut_writer.py | 142 ++++----- metagpt/utils/common.py | 43 ++- 9 files changed, 315 insertions(+), 376 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 49d2fe36f..8af137808 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -Provide configuration as a singleton. +Provide configuration, singleton """ import os import openai @@ -28,7 +28,7 @@ class NotConfiguredException(Exception): class Config(metaclass=Singleton): """ - Typical usage: + Common usage: config = Config("config.yaml") secret_key = config.get_key("MY_SECRET_KEY") print("Secret key:", secret_key) @@ -40,7 +40,7 @@ class Config(metaclass=Singleton): def __init__(self, yaml_file=default_yaml_file): self._configs = {} - self._initialize_with_config_files_and_environment(self._configs, yaml_file) + self._init_with_config_files_and_env(self._configs, yaml_file) logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") @@ -67,26 +67,26 @@ class Config(metaclass=Singleton): self.google_api_key = self._get("GOOGLE_API_KEY") self.google_cse_id = self._get("GOOGLE_CSE_ID") self.search_engine = self._get("SEARCH_ENGINE", SearchEngineType.SERPAPI_GOOGLE) - + self.web_browser_engine = WebBrowserEngineType(self._get("WEB_BROWSER_ENGINE", "playwright")) self.playwright_browser_type = self._get("PLAYWRIGHT_BROWSER_TYPE", "chromium") self.selenium_browser_type = self._get("SELENIUM_BROWSER_TYPE", "chrome") - + self.long_term_memory = self._get('LONG_TERM_MEMORY', False) if self.long_term_memory: logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 - def _initialize_with_config_files_and_environment(self, configs: dict, yaml_file): - """Load configurations from config/key.yaml, config/config.yaml, and the environment, in decreasing order of priority.""" + def _init_with_config_files_and_env(self, configs: dict, yaml_file): + """Load from config/key.yaml / config/config.yaml / env in decreasing priority""" configs.update(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: if not _yaml_file.exists(): continue - # Load local YAML files. + # Load local YAML file with open(_yaml_file, "r", encoding="utf-8") as file: yaml_data = yaml.safe_load(file) if not yaml_data: @@ -98,7 +98,7 @@ class Config(metaclass=Singleton): return self._configs.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """Fetch a value from config/key.yaml, config/config.yaml, or the environment. Raises an error if not found.""" + """Find values from config/key.yaml / config/config.yaml / env, report an error if not found""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") diff --git a/metagpt/const.py b/metagpt/const.py index 861da7903..c8ce80279 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -9,7 +9,7 @@ from pathlib import Path def get_project_root(): - """Search upwards level by level for the project root directory.""" + """Search upwards to find the project root directory.""" current_path = Path.cwd() while True: if (current_path / '.git').exists() or \ diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index baa10ba1e..906963aa1 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -28,7 +28,7 @@ class FaissStore(LocalStore): def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() if not (index_file.exists() and store_file.exists()): - logger.info("Missing at least one of index_file/store_file, load failed and return None") + logger.info("At least one of the index_file/store_file is missing. Loading failed and returns None.") return None index = faiss.read_index(str(index_file)) with open(str(store_file), "rb") as f: diff --git a/metagpt/document_store/milvus_store.py b/metagpt/document_store/milvus_store.py index 0a8ed78d4..175c04d13 100644 --- a/metagpt/document_store/milvus_store.py +++ b/metagpt/document_store/milvus_store.py @@ -19,9 +19,8 @@ type_mapping = { np.ndarray: DataType.FLOAT_VECTOR } - def columns_to_milvus_schema(columns: dict, primary_col_name: str = "", desc: str = ""): - """Assuming the structure of columns is str: regular type.""" + """Assuming the structure of columns is str: standard type""" fields = [] for col, ctype in columns.items(): if ctype == str: @@ -34,13 +33,11 @@ def columns_to_milvus_schema(columns: dict, primary_col_name: str = "", desc: st schema = CollectionSchema(fields, description=desc) return schema - class MilvusConnection(TypedDict): alias: str host: str port: str - class MilvusStore(BaseStore): """ FIXME: ADD TESTS @@ -79,8 +76,8 @@ class MilvusStore(BaseStore): """ FIXME: ADD TESTS https://milvus.io/docs/v2.0.x/search.md - All search and query operations within Milvus are executed in memory. Load the collection to memory before conducting a vector similarity search. - Noting the above description, is this logic serious? The time taken for this should be long, right? + All search and query operations within Milvus are executed in memory. Load the collection into memory before conducting a vector similarity search. + Noting the above description, is this logic serious? This should be time-consuming, right? """ search_params = {"metric_type": "L2", "params": {"nprobe": 10}} results = self.collection.search( @@ -91,7 +88,7 @@ class MilvusStore(BaseStore): expr=None, consistency_level="Strong" ) - # FIXME: results contains ids, but to get the actual values from the ids, the query interface still needs to be called. + # FIXME: results contain an id, but to get the actual value for the id, you still need to call the query interface return results def write(self, name, schema, *args, **kwargs): diff --git a/metagpt/prompts/summarize.py b/metagpt/prompts/summarize.py index a187314f4..348debf07 100644 --- a/metagpt/prompts/summarize.py +++ b/metagpt/prompts/summarize.py @@ -20,71 +20,72 @@ summary. Pick a suitable emoji for every bullet point. Your response should be i a YouTube video, use the following text: {{CONTENT}}. """ -# From GCP-VertexAI-Text Summarization +# From GCP-VertexAI-Text Summarization (SUMMARIZE_PROMPT_2-5 are all from this) # https://github.com/GoogleCloudPlatform/generative-ai/blob/main/language/examples/prompt-design/text_summarization.ipynb -# For longer documents, a map-reduce process is needed, see the following notebook +# Long documents need a map-reduce process, see the following notebook # https://github.com/GoogleCloudPlatform/generative-ai/blob/main/language/examples/document-summarization/summarization_large_documents.ipynb SUMMARIZE_PROMPT_2 = """ Provide a very short summary, no more than three sentences, for the following article: -Our quantum computers work by manipulating qubits in a manner we call quantum algorithms. -The challenge is that qubits are extremely sensitive, to the extent that even stray light can introduce calculation errors — a problem that intensifies as quantum computers scale. -This has notable ramifications since the most effective quantum algorithms we know for executing valuable applications necessitate that our qubits' error rates be significantly lower than current levels. -To address this discrepancy, quantum error correction is essential. -Quantum error correction safeguards information by distributing it over several physical qubits, forming a “logical qubit.” This is believed to be the sole method to create a large-scale quantum computer with sufficiently low error rates for practical calculations. -Rather than computing on individual qubits, we will utilize logical qubits. By transforming a greater number of physical qubits on our quantum processor into a single logical qubit, we aim to reduce error rates, enabling viable quantum algorithms. +Our quantum computers work by manipulating qubits in an orchestrated fashion that we call quantum algorithms. +The challenge is that qubits are so sensitive that even stray light can cause calculation errors — and the problem worsens as quantum computers grow. +This has significant consequences, since the best quantum algorithms that we know for running useful applications require the error rates of our qubits to be far lower than we have today. +To bridge this gap, we will need quantum error correction. +Quantum error correction protects information by encoding it across multiple physical qubits to form a “logical qubit,” and is believed to be the only way to produce a large-scale quantum computer with error rates low enough for useful calculations. +Instead of computing on the individual qubits themselves, we will then compute on logical qubits. By encoding larger numbers of physical qubits on our quantum processor into one logical qubit, we hope to reduce the error rates to enable useful quantum algorithms. Summary: """ + SUMMARIZE_PROMPT_3 = """ Provide a TL;DR for the following article: -Our quantum computers operate by controlling qubits in a method termed quantum algorithms. -The problem is that qubits are incredibly delicate, so much so that even minimal light interference can introduce computational errors — and this issue becomes more pronounced as quantum computers expand. -This is consequential because the most potent quantum algorithms we are aware of, for practical applications, demand that our qubits' error rates be substantially below current standards. -To mitigate this, quantum error correction is pivotal. -Quantum error correction secures data by distributing it across numerous physical qubits, generating a “logical qubit.” It's believed to be the exclusive approach to develop a large-scale quantum computer with error rates low enough for practical operations. -Instead of operations on individual qubits, we'll focus on logical qubits. By encoding a greater number of physical qubits on our quantum device into a single logical qubit, we aspire to diminish error rates and enable efficient quantum algorithms. +Our quantum computers work by manipulating qubits in an orchestrated fashion that we call quantum algorithms. +The challenge is that qubits are so sensitive that even stray light can cause calculation errors — and the problem worsens as quantum computers grow. +This has significant consequences, since the best quantum algorithms that we know for running useful applications require the error rates of our qubits to be far lower than we have today. +To bridge this gap, we will need quantum error correction. +Quantum error correction protects information by encoding it across multiple physical qubits to form a “logical qubit,” and is believed to be the only way to produce a large-scale quantum computer with error rates low enough for useful calculations. +Instead of computing on the individual qubits themselves, we will then compute on logical qubits. By encoding larger numbers of physical qubits on our quantum processor into one logical qubit, we hope to reduce the error rates to enable useful quantum algorithms. TL;DR: """ + SUMMARIZE_PROMPT_4 = """ Provide a very short summary in four bullet points for the following article: -Our quantum computers function by manipulating qubits through a method known as quantum algorithms. -The dilemma is that qubits are exceedingly fragile, so much so that even minimal light can lead to computational inaccuracies — and this problem amplifies as quantum computers become larger. -This is significant because the most proficient quantum algorithms known to us, suitable for real-world applications, necessitate that our qubits' error rates be significantly below what we currently observe. -To bridge this disparity, quantum error correction becomes indispensable. -Quantum error correction secures data by spreading it across multiple physical qubits, resulting in a “logical qubit.” It's perceived as the only technique to manufacture a large-scale quantum computer with error rates sufficiently low for practical tasks. -Instead of operating on individual qubits directly, we'll be utilizing logical qubits. By converting more physical qubits on our quantum machine into a single logical qubit, we intend to lower error rates, facilitating effective quantum algorithms. +Our quantum computers work by manipulating qubits in an orchestrated fashion that we call quantum algorithms. +The challenge is that qubits are so sensitive that even stray light can cause calculation errors — and the problem worsens as quantum computers grow. +This has significant consequences, since the best quantum algorithms that we know for running useful applications require the error rates of our qubits to be far lower than we have today. +To bridge this gap, we will need quantum error correction. +Quantum error correction protects information by encoding it across multiple physical qubits to form a “logical qubit,” and is believed to be the only way to produce a large-scale quantum computer with error rates low enough for useful calculations. +Instead of computing on the individual qubits themselves, we will then compute on logical qubits. By encoding larger numbers of physical qubits on our quantum processor into one logical qubit, we hope to reduce the error rates to enable useful quantum algorithms. Bulletpoints: """ + SUMMARIZE_PROMPT_5 = """ Please generate a summary of the following conversation and at the end summarize the to-do's for the support Agent: Customer: Hi, I'm Larry, and I received the wrong item. -Support Agent: Hi, Larry. How would you like this issue to be resolved? +Support Agent: Hi, Larry. How would you like to see this resolved? -Customer: That's alright. I'd like to return the item and receive a refund, please. +Customer: That's alright. I want to return the item and get a refund, please. -Support Agent: Certainly. I can process the refund for you right now. Could I have your order number, please? +Support Agent: Of course. I can process the refund for you now. Can I have your order number, please? Customer: It's [ORDER NUMBER]. -Support Agent: Thanks. I've processed the refund, and you should receive your funds within 14 days. +Support Agent: Thank you. I've processed the refund, and you will receive your money back within 14 days. -Customer: I appreciate it. +Customer: Thank you very much. -Support Agent: You're welcome, Larry. Have a great day! +Support Agent: You're welcome, Larry. Have a good day! Summary: -""" - -# - def summarize(doc: str) -> str # Input a document and receive a summary. +""" \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index a48f4fc9d..4b171917a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -1,243 +1,187 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/5 23:08 +@Time : 2023/5/11 14:43 @Author : alexanderwu -@File : openai.py +@File : engineer.py """ import asyncio -import time -from functools import wraps -from typing import NamedTuple +import shutil +from collections import OrderedDict +from pathlib import Path -import openai - -from metagpt.config import CONFIG +from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.utils.singleton import Singleton -from metagpt.utils.token_counter import ( - TOKEN_COSTS, - count_message_tokens, - count_string_tokens, -) +from metagpt.roles import Role +from metagpt.actions import WriteCode, WriteCodeReview, WriteTasks, WriteDesign +from metagpt.schema import Message +from metagpt.utils.common import CodeParser -def retry(max_retries): - def decorator(f): - @wraps(f) - async def wrapper(*args, **kwargs): - for i in range(max_retries): +async def gather_ordered_k(coros, k) -> list: + tasks = OrderedDict() + results = [None] * len(coros) + done_queue = asyncio.Queue() + + for i, coro in enumerate(coros): + if len(tasks) >= k: + done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED) + for task in done: + index = tasks.pop(task) + await done_queue.put((index, task.result())) + task = asyncio.create_task(coro) + tasks[task] = i + + if tasks: + done, _ = await asyncio.wait(tasks.keys()) + for task in done: + index = tasks[task] + await done_queue.put((index, task.result())) + + while not done_queue.empty(): + index, result = await done_queue.get() + results[index] = result + + return results + + +class Engineer(Role): + def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", + constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", + n_borg=1, use_code_review=False): + super().__init__(name, profile, goal, constraints) + self._init_actions([WriteCode]) + self.use_code_review = use_code_review + if self.use_code_review: + self._init_actions([WriteCode, WriteCodeReview]) + self._watch([WriteTasks]) + self.todos = [] + self.n_borg = n_borg + + @classmethod + def parse_tasks(cls, task_msg: Message) -> list[str]: + if not task_msg.instruct_content: + return task_msg.instruct_content.dict().get("Task list") + return CodeParser.parse_file_list(block="Task list", text=task_msg.content) + + @classmethod + def parse_code(cls, code_text: str) -> str: + return CodeParser.parse_code(block="", text=code_text) + + @classmethod + def parse_workspace(cls, system_design_msg: Message) -> str: + if not system_design_msg.instruct_content: + return system_design_msg.instruct_content.dict().get("Python package name") + return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) + + def get_workspace(self) -> Path: + msg = self._rc.memory.get_by_action(WriteDesign)[-1] + if not msg: + return WORKSPACE_ROOT / 'src' + workspace = self.parse_workspace(msg) + # Codes are written in workspace/{package_name}/{package_name} + return WORKSPACE_ROOT / workspace / workspace + + def recreate_workspace(self): + workspace = self.get_workspace() + try: + shutil.rmtree(workspace) + except FileNotFoundError: + pass # Directory does not exist, but we don't care + workspace.mkdir(parents=True, exist_ok=True) + + def write_file(self, filename: str, code: str): + workspace = self.get_workspace() + file = workspace / filename + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(code) + + def recv(self, message: Message) -> None: + self._rc.memory.add(message) + if message in self._rc.important_memory: + self.todos = self.parse_tasks(message) + + async def _act_mp(self) -> Message: + # self.recreate_workspace() + todo_coros = [] + for todo in self.todos: + todo_coro = WriteCode().run( + context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), + filename=todo + ) + todo_coros.append(todo_coro) + + rsps = await gather_ordered_k(todo_coros, self.n_borg) + for todo, code_rsp in zip(self.todos, rsps): + _ = self.parse_code(code_rsp) + logger.info(todo) + logger.info(code_rsp) + # self.write_file(todo, code) + msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) + self._rc.memory.add(msg) + del self.todos[0] + + logger.info(f'Done {self.get_workspace()} generating.') + msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) + return msg + + async def _act_sp(self) -> Message: + for todo in self.todos: + code_rsp = await WriteCode().run( + context=self._rc.history, + filename=todo + ) + # logger.info(todo) + # logger.info(code_rsp) + # code = self.parse_code(code_rsp) + self.write_file(todo, code_rsp) + msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) + self._rc.memory.add(msg) + + logger.info(f'Done {self.get_workspace()} generating.') + msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) + return msg + + async def _act_sp_precision(self) -> Message: + for todo in self.todos: + """ + # Select essential information from historical information to reduce prompt length (summarized from human experience) + 1. All from Architect + 2. All from ProjectManager + 3. Do you need other codes (currently needed)? + TODO: The goal is not to need it. Once tasks are split clearly, according to the design idea, the code can be written clearly for each file without other codes. If it can't, it means that it still needs to be defined more clearly, this is the key to write long code. + """ + context = [] + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + for m in msg: + context.append(m.content) + context_str = "\n".join(context) + # Writing code + code = await WriteCode().run( + context=context_str, + filename=todo + ) + # Code review + if self.use_code_review: try: - return await f(*args, **kwargs) - except Exception: - if i == max_retries - 1: - raise - await asyncio.sleep(2 ** i) - return wrapper - return decorator + rewrite_code = await WriteCodeReview().run( + context=context_str, + code=code, + filename=todo + ) + code = rewrite_code + except Exception as e: + logger.error("code review failed!", e) + pass + self.write_file(todo, code) + msg = Message(content=code, role=self.profile, cause_by=WriteCode) + self._rc.memory.add(msg) + logger.info(f'Done {self.get_workspace()} generating.') + msg = Message(content="all done.", role=self.profile, cause_by=WriteCode) + return msg -class RateLimiter: - """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed.""" - def __init__(self, rpm): - self.last_call_time = 0 - self.interval = 1.1 * 60 / rpm # Using 1.1 since strict adherence to time can still lead to QoS issues; consider simple error retry later. - self.rpm = rpm - - def split_batches(self, batch): - return [batch[i:i + self.rpm] for i in range(0, len(batch), self.rpm)] - - async def wait_if_needed(self, num_requests): - current_time = time.time() - elapsed_time = current_time - self.last_call_time - - if elapsed_time < self.interval * num_requests: - remaining_time = self.interval * num_requests - elapsed_time - logger.info(f"sleep {remaining_time}") - await asyncio.sleep(remaining_time) - - self.last_call_time = time.time() - - -class Costs(NamedTuple): - total_prompt_tokens: int - total_completion_tokens: int - total_cost: float - total_budget: float - - -class CostManager(metaclass=Singleton): - """Calculate the cost of using the API.""" - def __init__(self): - self.total_prompt_tokens = 0 - self.total_completion_tokens = 0 - self.total_cost = 0 - self.total_budget = 0 - - def update_cost(self, prompt_tokens, completion_tokens, model): - """ - Update the total cost, prompt tokens, and completion tokens. - - Args: - prompt_tokens (int): The number of tokens used in the prompt. - completion_tokens (int): The number of tokens used in the completion. - model (str): The model used for the API call. - """ - self.total_prompt_tokens += prompt_tokens - self.total_completion_tokens += completion_tokens - cost = ( - prompt_tokens * TOKEN_COSTS[model]["prompt"] - + completion_tokens * TOKEN_COSTS[model]["completion"] - ) / 1000 - self.total_cost += cost - logger.info(f"Total running cost: ${self.total_cost:.3f} | Max budget: ${CONFIG.max_budget:.3f} | " - f"Current cost: ${cost:.3f}, {prompt_tokens=}, {completion_tokens=}") - CONFIG.total_cost = self.total_cost - - def get_total_prompt_tokens(self): - """Get the total number of prompt tokens.""" - return self.total_prompt_tokens - - def get_total_completion_tokens(self): - """Get the total number of completion tokens.""" - return self.total_completion_tokens - - def get_total_cost(self): - """Get the total cost of API calls.""" - return self.total_cost - - def get_costs(self) -> Costs: - """Get all costs.""" - return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) - - -class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): - """ - Check https://platform.openai.com/examples for examples. - """ - def __init__(self): - self.__init_openai(CONFIG) - self.llm = openai - self.model = CONFIG.openai_api_model - self._cost_manager = CostManager() - RateLimiter.__init__(self, rpm=self.rpm) - - def __init_openai(self, config): - openai.api_key = config.openai_api_key - if config.openai_api_base: - openai.api_base = config.openai_api_base - if config.openai_api_type: - openai.api_type = config.openai_api_type - openai.api_version = config.openai_api_version - self.rpm = int(config.get("RPM", 10)) - - async def _achat_completion_stream(self, messages: list[dict]) -> str: - response = await openai.ChatCompletion.acreate( - **self._cons_kwargs(messages), - stream=True - ) - - # create variables to collect the stream of chunks - collected_chunks = [] - collected_messages = [] - # iterate through the stream of events - async for chunk in response: - collected_chunks.append(chunk) # save the event response - chunk_message = chunk['choices'][0]['delta'] # extract the message - collected_messages.append(chunk_message) # save the message - if "content" in chunk_message: - print(chunk_message["content"], end="") - print() - - full_reply_content = ''.join([m.get('content', '') for m in collected_messages]) - usage = self._calc_usage(messages, full_reply_content) - self._update_costs(usage) - return full_reply_content - - def _cons_kwargs(self, messages: list[dict]) -> dict: - if CONFIG.openai_api_type == 'azure': - kwargs = { - "deployment_id": CONFIG.deployment_id, - "messages": messages, - "max_tokens": CONFIG.max_tokens_rsp, - "n": 1, - "stop": None, - "temperature": 0.3 - } - else: - kwargs = { - "model": self.model, - "messages": messages, - "max_tokens": CONFIG.max_tokens_rsp, - "n": 1, - "stop": None, - "temperature": 0.3 - } - return kwargs - - async def _achat_completion(self, messages: list[dict]) -> dict: - rsp = await self.llm.ChatCompletion.acreate(**self._cons_kwargs(messages)) - self._update_costs(rsp.get('usage')) - return rsp - - def _chat_completion(self, messages: list[dict]) -> dict: - rsp = self.llm.ChatCompletion.create(**self._cons_kwargs(messages)) - self._update_costs(rsp) - return rsp - - def completion(self, messages: list[dict]) -> dict: - return self._chat_completion(messages) - - async def acompletion(self, messages: list[dict]) -> dict: - return await self._achat_completion(messages) - - @retry(max_retries=6) - async def acompletion_text(self, messages: list[dict], stream=False) -> str: - """When streaming, print each token in place.""" - if stream: - return await self._achat_completion_stream(messages) - rsp = await self._achat_completion(messages) - return self.get_choice_text(rsp) - - def _calc_usage(self, messages: list[dict], rsp: str) -> dict: - usage = {} - prompt_tokens = count_message_tokens(messages, self.model) - completion_tokens = count_string_tokens(rsp, self.model) - usage['prompt_tokens'] = prompt_tokens - usage['completion_tokens'] = completion_tokens - return usage - - async def acompletion_batch(self, batch: list[list[dict]]) -> list[dict]: - """Returns the full JSON.""" - split_batches = self.split_batches(batch) - all_results = [] - - for small_batch in split_batches: - logger.info(small_batch) - await self.wait_if_needed(len(small_batch)) - - future = [self.acompletion(prompt) for prompt in small_batch] - results = await asyncio.gather(*future) - logger.info(results) - all_results.extend(results) - - return all_results - - async def acompletion_batch_text(self, batch: list[list[dict]]) -> list[str]: - """Returns only plain text.""" - raw_results = await self.acompletion_batch(batch) - results = [] - for idx, raw_result in enumerate(raw_results, start=1): - result = self.get_choice_text(raw_result) - results.append(result) - logger.info(f"Result of task {idx}: {result}") - return results - - def _update_costs(self, usage: dict): - prompt_tokens = int(usage['prompt_tokens']) - completion_tokens = int(usage['completion_tokens']) - self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) - - def get_costs(self) -> Costs: - return self._cost_manager.get_costs() + async def _act(self) -> Message: + if self.use_code_review: + return await self._act_sp_precision() + return await self._act_sp() diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index aa776f662..606952b99 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/19 16:28 # @Author : stellahong (stellahong@fuzhi.ai) -# @Desc : +# @Description : + import os import asyncio from os.path import join @@ -55,6 +56,7 @@ payload = { default_negative_prompt = "(easynegative:0.8),black, dark,Low resolution" + class SDEngine: def __init__(self): # Initialize the SDEngine with configuration @@ -65,7 +67,7 @@ class SDEngine: self.payload = payload logger.info(self.sd_t2i_url) - def construct_payload(self, prompt, negtive_prompt=default_negative_prompt, width=512, height=512, + def construct_payload(self, prompt, negative_prompt=default_negative_prompt, width=512, height=512, sd_model="galaxytimemachinesGTM_photoV20"): # Configure the payload with provided inputs self.payload["prompt"] = prompt @@ -101,11 +103,11 @@ class SDEngine: return imgs async def run_i2i(self): - # TODO: Add image-to-image API call + # TODO: Add a method to call the image-to-image interface raise NotImplementedError async def run_sam(self): - # TODO: Add SAM API call + # TODO: Add a method to call the SAM interface raise NotImplementedError def decode_base64_to_image(img, save_name): diff --git a/metagpt/tools/ut_writer.py b/metagpt/tools/ut_writer.py index aca335246..23604ac54 100644 --- a/metagpt/tools/ut_writer.py +++ b/metagpt/tools/ut_writer.py @@ -6,18 +6,18 @@ from pathlib import Path from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI -ICL_SAMPLE = '''API Definition: +ICL_SAMPLE = '''Interface definition: ```text -API Name: Tag Elements -API Path: /projects/{project_key}/node-tags -Method: POST +Interface Name: Tag Elements +Interface Path: /projects/{project_key}/node-tags +Method: POST Request Parameters: Path Parameters: project_key Body Parameters: -Name Type Required Default Value Description +Name Type Required Default Value Description nodes array Yes Nodes node_key string No Node key tags array No Original node tag list @@ -26,11 +26,11 @@ operations array Yes tags array No Operation tag list mode string No Operation type ADD / DELETE -Response Data: -Name Type Required Default Value Description +Return Data: +Name Type Required Default Value Description code integer Yes Status code msg string Yes Message -data object Yes Response data +data object Yes Return data list array No Node list true / false node_type string No Node type DATASET / RECIPE node_key string No Node key @@ -43,7 +43,7 @@ Unit Test: [ ("project_key", [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "success"), ("project_key", [{"node_key": "dataset_002", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["tag1"], "mode": "DELETE"}], "success"), -("", [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Missing required parameter project_key"), +("", [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Missing necessary parameter project_key"), (123, [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Incorrect parameter type"), ("project_key", [{"node_key": "a"*201, "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Request parameter exceeds field boundary") ] @@ -51,64 +51,67 @@ Unit Test: def test_node_tags(project_key, nodes, operations, expected_msg): pass ``` -Above is an example of an API definition and a unit test sample. -Next, please play the role of a test manager from Google with 20 years of experience. After I provide the API definition, reply with the unit test. There are a few requirements: -1. Only output one '@pytest.mark.parametrize' and its corresponding 'test_' function (with only a 'pass' statement inside, no implementation). --- The function parameters should include 'expected_msg' for result validation. -2. The generated test cases should use shorter text or numbers and be as compact as possible. +The above is an example of interface definition and unit test. +Next, please act as an expert test manager with 20 years of experience at Google. +After I provide the interface definition, please reply with the unit test. +There are a few requirements: +1. Only output one `@pytest.mark.parametrize` and the corresponding test_ function + (with a pass inside, not implemented). + -- The function parameters should include expected_msg for result validation. +2. The generated test cases should use shorter text or numbers and be as concise as possible. 3. If comments are needed, use Chinese. -If you understand, please wait for me to provide the API definition and only reply with "Understood" to save tokens. +If you understand, please wait for me to provide the interface definition +and only reply with "Understood" to save tokens. ''' -ACT_PROMPT_PREFIX = '''Reference test types: such as missing request parameters, field boundary validation, incorrect field type. +ACT_PROMPT_PREFIX = '''Reference test types: such as missing request parameters, field boundary checks, incorrect field types. Please output 10 test cases within a `@pytest.mark.parametrize` scope. ```text ''' -YFT_PROMPT_PREFIX = '''Reference test types: such as SQL injection, cross-site scripting (XSS), illegal access and unauthorized access, authentication and authorization, parameter validation, exception handling, file upload and download. +YFT_PROMPT_PREFIX = '''Reference test types: such as SQL injection, cross-site scripting (XSS), illegal access and unauthorized access, authentication and authorization, parameter verification, exception handling, file upload and download. Please output 10 test cases within a `@pytest.mark.parametrize` scope. ```text ''' - OCR_API_DOC = '''```text -API Name: OCR Recognition -API Path: /api/v1/contract/treaty/task/ocr -Method: POST +API Name: OCR Recognition +API Path: /api/v1/contract/treaty/task/ocr +Method: POST Request Parameters: Path Parameters: Body Parameters: -Name Type Required Default Value Remarks -file_id string Yes -box array Yes -contract_id number Yes Contract ID -start_time string No yyyy-mm-dd -end_time string No yyyy-mm-dd -extract_type number No Recognition type 1-During import 2-After import, default is 1 +Name Type Mandatory Default Value Remarks +file_id string Yes +box array Yes +contract_id number Yes Contract ID +start_time string No yyyy-mm-dd +end_time string No yyyy-mm-dd +extract_type number No Recognition Type 1- During Import 2- After Import, Default is 1 -Response Data: -Name Type Required Default Value Remarks -code integer Yes -message string Yes -data object Yes +Return Data: +Name Type Mandatory Default Value Remarks +code integer Yes +message string Yes +data object Yes + ''' - class UTGenerator: - """UT Generator: Construct UT through API documentation.""" + """UT Generator: Constructs UT from API documentation.""" def __init__(self, swagger_file: str, ut_py_path: str, questions_path: str, chatgpt_method: str = "API", template_prefix=YFT_PROMPT_PREFIX) -> None: - """Initialize the UT Generator. + """Initializes the UT generator. Args: - swagger_file: Path to the swagger file. + swagger_file: Path to the swagger. ut_py_path: Path to store test cases. - questions_path: Path to store templates for future investigation. - chatgpt_method: API - template_prefix: Use template, default is YFT_UT_PROMPT. + questions_path: Path to store templates for further investigation. + chatgpt_method: API. + template_prefix: Use template, defaults to YFT_UT_PROMPT. """ self.swagger_file = swagger_file self.ut_py_path = ut_py_path @@ -116,56 +119,56 @@ class UTGenerator: assert chatgpt_method in ["API"], "Invalid chatgpt_method" self.chatgpt_method = chatgpt_method - # ICL: In-Context Learning. Provide an example here for GPT to mimic. + # ICL: In-Context Learning. Here, an example is provided for GPT to follow. self.icl_sample = ICL_SAMPLE self.template_prefix = template_prefix def get_swagger_json(self) -> dict: - """Load Swagger JSON from a local file.""" + """Loads Swagger JSON from a local file.""" with open(self.swagger_file, "r", encoding="utf-8") as file: swagger_json = json.load(file) return swagger_json - def __parameter_to_string(self, prop, required, name=""): + def __para_to_str(self, prop, required, name=""): name = name or prop["name"] ptype = prop["type"] title = prop.get("title", "") desc = prop.get("description", "") return f'{name}\t{ptype}\t{"Yes" if required else "No"}\t{title}\t{desc}' - def _parameter_to_string(self, prop): + def _para_to_str(self, prop): required = prop.get("required", False) - return self.__parameter_to_string(prop, required) + return self.__para_to_str(prop, required) - def parameter_to_string(self, name, prop, prop_object_required): + def para_to_str(self, name, prop, prop_object_required): required = name in prop_object_required - return self.__parameter_to_string(prop, required, name) + return self.__para_to_str(prop, required, name) def build_object_properties(self, node, prop_object_required, level: int = 0) -> str: - """Recursively output properties of object and array[object] types. + """Recursively outputs properties of object and array[object] types. Args: - node (_type_): Value of the sub-item. - prop_object_required (_type_): Indicates if it's a required item. + node: Value of the sub-item. + prop_object_required: Whether it's a required item. level: Current recursion depth. """ doc = "" def dive_into_object(node): - """If it's an object type, recursively output its properties.""" + """If it's an object type, recursively outputs its properties.""" if node.get("type") == "object": sub_properties = node.get("properties", {}) return self.build_object_properties(sub_properties, prop_object_required, level=level + 1) return "" if node.get("in", "") in ["query", "header", "formData"]: - doc += f'{"\t" * level}{self._parameter_to_string(node)}\n' + doc += f'{"\t" * level}{self._para_to_str(node)}\n' doc += dive_into_object(node) return doc for name, prop in node.items(): - doc += f'{"\t" * level}{self.parameter_to_string(name, prop, prop_object_required)}\n' + doc += f'{"\t" * level}{self.para_to_str(name, prop, prop_object_required)}\n' doc += dive_into_object(prop) if prop["type"] == "array": items = prop.get("items", {}) @@ -173,10 +176,10 @@ class UTGenerator: return doc def get_tags_mapping(self) -> dict: - """Process tag and path. + """Handles tag and path mapping. Returns: - Dictionary: Correspondence of tag to path. + Dict: Mapping of tag to path. """ swagger_data = self.get_swagger_json() paths = swagger_data["paths"] @@ -194,7 +197,7 @@ class UTGenerator: return tags def generate_ut(self, include_tags) -> bool: - """Generate test case files.""" + """Generates test case files.""" tags = self.get_tags_mapping() for tag, paths in tags.items(): if include_tags is None or tag in include_tags: @@ -204,19 +207,19 @@ class UTGenerator: def build_api_doc(self, node: dict, path: str, method: str) -> str: summary = node["summary"] - doc = f"API Name: {summary}\nAPI Path: {path}\nMethod: {method.upper()}\n" - doc += "\nRequest Parameters:\n" + doc = f"Interface name: {summary}\nInterface path: {path}\nMethod: {method.upper()}\n" + doc += "\nRequest parameters:\n" if "parameters" in node: parameters = node["parameters"] - doc += "Path Parameters:\n" + doc += "Path parameters:\n" # param["in"]: path / formData / body / query / header for param in parameters: if param["in"] == "path": - doc += f'{param["name"]}\n' + doc += f'{param["name"]} \n' - doc += "\nBody Parameters:\n" - doc += "Name\tType\tRequired\tDefault Value\tRemarks\n" + doc += "\nBody parameters:\n" + doc += "Name\tType\tRequired\tDefault\tNotes\n" for param in parameters: if param["in"] == "body": schema = param.get("schema", {}) @@ -227,8 +230,8 @@ class UTGenerator: doc += self.build_object_properties(param, []) # Output return data information - doc += "\nReturn Data:\n" - doc += "Name\tType\tRequired\tDefault Value\tRemarks\n" + doc += "\nReturn data:\n" + doc += "Name\tType\tRequired\tDefault\tNotes\n" responses = node["responses"] response = responses.get("200", {}) schema = response.get("schema", {}) @@ -242,12 +245,13 @@ class UTGenerator: return doc def _store(self, data, base, folder, fname): + """Store data in a file.""" file_path = self.get_file_path(Path(base) / folder, fname) with open(file_path, "w", encoding="utf-8") as file: file.write(data) def ask_gpt_and_save(self, question: str, tag: str, fname: str): - """Generate a question and store both question and answer.""" + """Generate a question and save both the question and answer.""" messages = [self.icl_sample, question] result = self.gpt_msgs_to_code(messages=messages) @@ -255,11 +259,11 @@ class UTGenerator: self._store(result, self.ut_py_path, tag, f"{fname}.py") def _generate_ut(self, tag, paths): - """Handle structure under the data path. + """Process the structure under the data path. Args: - tag (_type_): Module name. - paths (_type_): Path Object. + tag: Module name. + paths: Path Object. """ for path, path_obj in paths.items(): for method, node in path_obj.items(): @@ -269,7 +273,7 @@ class UTGenerator: self.ask_gpt_and_save(question, tag, summary) def gpt_msgs_to_code(self, messages: list) -> str: - """Choose based on different invocation methods.""" + """Choose based on different call methods.""" result = '' if self.chatgpt_method == "API": result = GPTAPI().ask_code(msgs=messages) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index fb91d2c57..aa2f5bb98 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -14,10 +14,11 @@ from typing import List, Tuple from metagpt.logs import logger -def check_command_exists(command) -> int: - """ Check if a command exists. +def check_cmd_exists(command) -> int: + """Check if a command exists. + :param command: Command to check. - :return: Returns 0 if the command exists, else returns non-zero. + :return: Returns 0 if the command exists, otherwise returns a non-zero value. """ check_command = 'command -v ' + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }' result = os.system(check_command) @@ -28,19 +29,19 @@ class OutputParser: @classmethod def parse_blocks(cls, text: str): - # Firstly, split the text into different blocks based on "##". + # First, split the text into different blocks based on "##". blocks = text.split("##") # Create a dictionary to store the title and content of each block. block_dict = {} - # Loop through all blocks. + # Iterate through all blocks. for block in blocks: - # If block is not empty, continue processing. + # If the block is not empty, continue processing. if block.strip() != "": - # Split block's title and content and trim them. + # Split the block's title and content and trim whitespace. block_title, block_content = block.split("\n", 1) - # There may be errors in LLM, correct it here. + # LLM might make mistakes; correct it here. if block_title[-1] == ":": block_title = block_title[:-1] block_dict[block_title.strip()] = block_content.strip() @@ -84,7 +85,7 @@ class OutputParser: block_dict = cls.parse_blocks(data) parsed_data = {} for block, content in block_dict.items(): - # Try to remove code markers. + # Try to remove the code marker. try: content = cls.parse_code(text=content) except Exception: @@ -103,7 +104,7 @@ class OutputParser: block_dict = cls.parse_blocks(data) parsed_data = {} for block, content in block_dict.items(): - # Try to remove code markers. + # Try to remove the code marker. try: content = cls.parse_code(text=content) except Exception: @@ -135,17 +136,17 @@ class CodeParser: @classmethod def parse_blocks(cls, text: str): - # Firstly, split the text into different blocks based on "##". + # First, split the text into different blocks based on "##". blocks = text.split("##") # Create a dictionary to store the title and content of each block. block_dict = {} - # Loop through all blocks. + # Iterate through all blocks. for block in blocks: - # If block is not empty, continue processing. + # If the block is not empty, continue processing. if block.strip() != "": - # Split block's title and content and trim them. + # Split the block's title and content and trim whitespace. block_title, block_content = block.split("\n", 1) block_dict[block_title.strip()] = block_content.strip() @@ -160,7 +161,7 @@ class CodeParser: if match: code = match.group(1) else: - logger.error(f"{pattern} did not match the following text:") + logger.error(f"{pattern} not match following text:") logger.error(text) raise Exception return code @@ -213,14 +214,4 @@ def print_members(module, indent=0): prefix = ' ' * indent for name, obj in inspect.getmembers(module): print(name, obj) - if inspect.isclass(obj): - print(f'{prefix}Class: {name}') - # print the methods within the class - if name in ['__class__', '__base__']: - continue - print_members(obj, indent + 2) - elif inspect.isfunction(obj): - print(f'{prefix}Function: {name}') - elif inspect.ismethod(obj): - print(f'{prefix}Method: {name}') - + if inspect From 0eea6bc19f6f76c973be36266a6042d67bdd13f9 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Tue, 1 Aug 2023 20:56:09 -0500 Subject: [PATCH 04/27] corrected the common.py file --- metagpt/utils/common.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index aa2f5bb98..399ff22de 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -214,4 +214,13 @@ def print_members(module, indent=0): prefix = ' ' * indent for name, obj in inspect.getmembers(module): print(name, obj) - if inspect + if inspect.isclass(obj): + print(f'{prefix}Class: {name}') + # print the methods within the class + if name in ['__class__', '__base__']: + continue + print_members(obj, indent + 2) + elif inspect.isfunction(obj): + print(f'{prefix}Function: {name}') + elif inspect.ismethod(obj): + print(f'{prefix}Method: {name}') \ No newline at end of file From 0d632c7e567764b302af2f805c0f4058fee55f1c Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 2 Aug 2023 15:57:10 -0500 Subject: [PATCH 05/27] another run through for translation. --- .gitignore | 2 +- Dockerfile | 2 +- Message | 0 None | 0 config/config.yaml | 17 +- int | 0 metagpt/actions/__init__.py | 1 + metagpt/actions/action.py | 1 + metagpt/actions/action_output.py | 1 + metagpt/actions/analyze_dep_libs.py | 4 +- metagpt/actions/azure_tts.py | 4 +- metagpt/actions/debug_error.py | 1 + metagpt/actions/design_api.py | 3 +- metagpt/actions/design_api_review.py | 1 + metagpt/actions/design_filenames.py | 1 + metagpt/actions/project_management.py | 1 + metagpt/actions/run_code.py | 1 + metagpt/actions/search_and_summarize.py | 1 + metagpt/actions/write_code.py | 1 + metagpt/actions/write_code_review.py | 1 + metagpt/actions/write_prd.py | 1 + metagpt/actions/write_prd_review.py | 1 + metagpt/actions/write_test.py | 1 + metagpt/config.py | 12 +- metagpt/document_store/base_store.py | 3 +- metagpt/document_store/chromadb_store.py | 4 +- metagpt/document_store/document.py | 1 + metagpt/document_store/faiss_store.py | 15 +- metagpt/document_store/milvus_store.py | 11 +- metagpt/environment.py | 14 +- metagpt/inspect_module.py | 2 +- metagpt/llm.py | 2 +- metagpt/management/skill_manager.py | 18 +- metagpt/manager.py | 4 +- metagpt/memory/longterm_memory.py | 1 + metagpt/memory/memory.py | 1 + metagpt/memory/memory_storage.py | 1 + metagpt/prompts/generate_skill.md | 29 +- metagpt/prompts/metagpt_sample.py | 38 +- metagpt/prompts/summarize.py | 13 +- metagpt/provider/anthropic_api.py | 1 + metagpt/provider/base_chatbot.py | 1 + metagpt/provider/base_gpt_api.py | 1 + metagpt/provider/openai_api.py | 422 ++++++++++-------- metagpt/roles/architect.py | 1 + metagpt/roles/customer_service.py | 1 + metagpt/roles/engineer.py | 89 ++-- metagpt/roles/product_manager.py | 1 + metagpt/roles/project_manager.py | 1 + metagpt/roles/prompt.py | 37 +- metagpt/roles/qa_engineer.py | 2 +- metagpt/roles/role.py | 160 +++---- metagpt/roles/sales.py | 1 + metagpt/roles/seacher.py | 1 + metagpt/schema.py | 6 +- metagpt/software_company.py | 1 + metagpt/tools/prompt_writer.py | 24 +- metagpt/tools/sd_engine.py | 39 +- metagpt/tools/search_engine.py | 35 +- metagpt/tools/search_engine_meilisearch.py | 2 +- metagpt/tools/search_engine_serpapi.py | 1 + metagpt/tools/search_engine_serper.py | 4 +- metagpt/tools/translator.py | 18 +- metagpt/tools/ut_writer.py | 146 +++--- metagpt/tools/web_browser_engine.py | 1 + .../tools/web_browser_engine_playwright.py | 2 +- metagpt/tools/web_browser_engine_selenium.py | 1 + metagpt/utils/common.py | 44 +- metagpt/utils/mermaid.py | 13 +- metagpt/utils/read_document.py | 6 +- metagpt/utils/singleton.py | 1 + startup.py | 1 + 72 files changed, 704 insertions(+), 575 deletions(-) create mode 100644 Message create mode 100644 None create mode 100644 int diff --git a/.gitignore b/.gitignore index c4c79c733..6abc90b43 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,4 @@ examples/nb/ workspace/* *.mmd tmp -output.wav +output.wav \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e8f717b7c..120b70442 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,4 +23,4 @@ RUN cd /app/metagpt &&\ WORKDIR /app/metagpt # Running with an infinite loop using the tail command -CMD ["sh", "-c", "tail -f /dev/null"] +CMD ["sh", "-c", "tail -f /dev/null"] \ No newline at end of file diff --git a/Message b/Message new file mode 100644 index 000000000..e69de29bb diff --git a/None b/None new file mode 100644 index 000000000..e69de29bb diff --git a/config/config.yaml b/config/config.yaml index 25301fa0f..ceab18854 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -2,10 +2,12 @@ # The configuration of key.yaml has a higher priority and will not enter git #### if OpenAI - -#OPENAI_API_KEY: "YOUR_API_KEY" -#OPENAI_API_BASE: "YOUR_API_BASE" +## The official OPENAI_API_BASE is https://api.openai.com/v1 +## If the official OPENAI_API_BASE is not available, we recommend using the [openai-forward](https://github.com/beidongjiedeguang/openai-forward). +## Or, you can configure OPENAI_PROXY to access official OPENAI_API_BASE. +OPENAI_API_BASE: "https://api.openai.com/v1" #OPENAI_PROXY: "http://127.0.0.1:8118" +#OPENAI_API_KEY: "YOUR_API_KEY" OPENAI_API_MODEL: "gpt-4" MAX_TOKENS: 1500 RPM: 10 @@ -55,3 +57,12 @@ SD_T2I_API: "/sdapi/v1/txt2img" #### for Execution #LONG_TERM_MEMORY: false + +#### for Mermaid CLI +## If you installed mmdc (Mermaid CLI) only for metagpt then enable the following configuration. +#PUPPETEER_CONFIG: "./config/puppeteer-config.json" +#MMDC: "./node_modules/.bin/mmdc" + +### for update_costs & calc_usage +UPDATE_COSTS: false +CALC_USAGE: false \ No newline at end of file diff --git a/int b/int new file mode 100644 index 000000000..e69de29bb diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index 0c861aa69..165349728 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -40,3 +40,4 @@ class ActionType(Enum): WRITE_TASKS = WriteTasks ASSIGN_TASKS = AssignTasks SEARCH_AND_SUMMARIZE = SearchAndSummarize + \ No newline at end of file diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index fa0d592a3..edd21d320 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -65,3 +65,4 @@ class Action(ABC): async def run(self, *args, **kwargs): """Run action""" raise NotImplementedError("The run method should be implemented in a subclass.") + \ No newline at end of file diff --git a/metagpt/actions/action_output.py b/metagpt/actions/action_output.py index c0b88dcf9..ea7f4fb80 100644 --- a/metagpt/actions/action_output.py +++ b/metagpt/actions/action_output.py @@ -40,3 +40,4 @@ class ActionOutput: new_class.__validator_check_name = classmethod(check_name) new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields) return new_class + \ No newline at end of file diff --git a/metagpt/actions/analyze_dep_libs.py b/metagpt/actions/analyze_dep_libs.py index 23c35cdf8..53d40200a 100644 --- a/metagpt/actions/analyze_dep_libs.py +++ b/metagpt/actions/analyze_dep_libs.py @@ -28,10 +28,10 @@ Focus only on the names of shared dependencies, do not add any other explanation class AnalyzeDepLibs(Action): def __init__(self, name, context=None, llm=None): super().__init__(name, context, llm) - self.desc = "根据上下文,分析程序运行依赖库" + self.desc = "Analyze the runtime dependencies of the program based on the context" async def run(self, requirement, filepaths_string): - # prompt = f"以下是产品需求文档(PRD):\n\n{prd}\n\n{PROMPT}" + # prompt = f"Below is the product requirement document (PRD):\n\n{prd}\n\n{PROMPT}" prompt = PROMPT.format(prompt=requirement, filepaths_string=filepaths_string) design_filenames = await self._aask(prompt) return design_filenames diff --git a/metagpt/actions/azure_tts.py b/metagpt/actions/azure_tts.py index f528ba001..c13a4750d 100644 --- a/metagpt/actions/azure_tts.py +++ b/metagpt/actions/azure_tts.py @@ -16,7 +16,7 @@ class AzureTTS(Action): super().__init__(name, context, llm) self.config = Config() - # 参数参考:https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles + # Parameters reference: https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles def synthesize_speech(self, lang, voice, role, text, output_file): subscription_key = self.config.get('AZURE_TTS_SUBSCRIPTION_KEY') region = self.config.get('AZURE_TTS_REGION') @@ -49,5 +49,5 @@ if __name__ == "__main__": "zh-CN", "zh-CN-YunxiNeural", "Boy", - "你好,我是卡卡", + "Hello, I am Kaka", "output.wav") diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index cd6cc4e36..6a7cbc872 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -17,3 +17,4 @@ class DebugError(Action): f"\n\n{error}\n\nPlease try to fix the error in this code." fixed_code = await self._aask(prompt) return fixed_code + \ No newline at end of file diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 1447eacc3..abd1f9d4c 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -100,7 +100,7 @@ class WriteDesign(Action): try: shutil.rmtree(workspace) except FileNotFoundError: - pass # 文件夹不存在,但我们不在意 + pass # Folder does not exist, but we don't care workspace.mkdir(parents=True, exist_ok=True) def _save_prd(self, docs_path, resources_path, prd): @@ -141,3 +141,4 @@ class WriteDesign(Action): system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) self._save(context, system_design) return system_design + \ No newline at end of file diff --git a/metagpt/actions/design_api_review.py b/metagpt/actions/design_api_review.py index 687a33652..9bb822a62 100644 --- a/metagpt/actions/design_api_review.py +++ b/metagpt/actions/design_api_review.py @@ -19,3 +19,4 @@ class DesignReview(Action): api_review = await self._aask(prompt) return api_review + \ No newline at end of file diff --git a/metagpt/actions/design_filenames.py b/metagpt/actions/design_filenames.py index 6c3d8e803..29400e950 100644 --- a/metagpt/actions/design_filenames.py +++ b/metagpt/actions/design_filenames.py @@ -26,3 +26,4 @@ class DesignFilenames(Action): logger.debug(prompt) logger.debug(design_filenames) return design_filenames + \ No newline at end of file diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 89c59dcda..3096eb94b 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -126,3 +126,4 @@ class AssignTasks(Action): async def run(self, *args, **kwargs): # Here you should implement the actual action pass + \ No newline at end of file diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 9a4de6d07..5b3106f4d 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -23,3 +23,4 @@ class RunCode(Action): except Exception: # If there is an error in the code, return the error message return traceback.format_exc() + \ No newline at end of file diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 43dc02838..945308689 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -137,3 +137,4 @@ class SearchAndSummarize(Action): logger.debug(prompt) logger.debug(result) return result + \ No newline at end of file diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index cc122ef7a..c000805c5 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -79,3 +79,4 @@ class WriteCode(Action): # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) return code + \ No newline at end of file diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 7f6a7a38e..4ff4d6cf6 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -79,3 +79,4 @@ class WriteCodeReview(Action): # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) return code + \ No newline at end of file diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 0edd24d55..2b96f867c 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -144,3 +144,4 @@ class WritePRD(Action): logger.debug(prompt) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) return prd + \ No newline at end of file diff --git a/metagpt/actions/write_prd_review.py b/metagpt/actions/write_prd_review.py index 5ff9624c5..5c922d3bc 100644 --- a/metagpt/actions/write_prd_review.py +++ b/metagpt/actions/write_prd_review.py @@ -25,3 +25,4 @@ class WritePRDReview(Action): prompt = self.prd_review_prompt_template.format(prd=self.prd) review = await self._aask(prompt) return review + \ No newline at end of file diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 25b53dac5..393b5fcd9 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -24,3 +24,4 @@ class WriteTest(Action): prompt = self.test_prompt_template.format(code=self.code) test_cases = await self._aask(prompt) return test_cases + \ No newline at end of file diff --git a/metagpt/config.py b/metagpt/config.py index 8af137808..e479ab018 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -28,9 +28,9 @@ class NotConfiguredException(Exception): class Config(metaclass=Singleton): """ - Common usage: + Regular usage method: config = Config("config.yaml") - secret_key = config.get_key("MY_SECRET_KEY") + secret_key = config.get("MY_SECRET_KEY") print("Secret key:", secret_key) """ @@ -77,9 +77,11 @@ class Config(metaclass=Singleton): logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 + self.puppeteer_config = self._get("PUPPETEER_CONFIG","") + self.mmdc = self._get("MMDC","mmdc") def _init_with_config_files_and_env(self, configs: dict, yaml_file): - """Load from config/key.yaml / config/config.yaml / env in decreasing priority""" + """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" configs.update(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: @@ -98,11 +100,11 @@ class Config(metaclass=Singleton): return self._configs.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """Find values from config/key.yaml / config/config.yaml / env, report an error if not found""" + """Search for a value in config/key.yaml, config/config.yaml, and env; raise an error if not found""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value -CONFIG = Config() +CONFIG = Config() \ No newline at end of file diff --git a/metagpt/document_store/base_store.py b/metagpt/document_store/base_store.py index 01877e106..27b5d94e0 100644 --- a/metagpt/document_store/base_store.py +++ b/metagpt/document_store/base_store.py @@ -12,7 +12,7 @@ from metagpt.config import Config class BaseStore(ABC): - """FIXME: consider add_index, set_index and think 颗粒度""" + """FIXME: consider add_index, set_index and think about granularity.""" @abstractmethod def search(self, query, *args, **kwargs): @@ -53,3 +53,4 @@ class LocalStore(BaseStore, ABC): @abstractmethod def _write(self, docs, metadatas): raise NotImplementedError + \ No newline at end of file diff --git a/metagpt/document_store/chromadb_store.py b/metagpt/document_store/chromadb_store.py index ee14fb2f0..d2ecc05f6 100644 --- a/metagpt/document_store/chromadb_store.py +++ b/metagpt/document_store/chromadb_store.py @@ -9,7 +9,7 @@ import chromadb class ChromaStore: - """如果从BaseStore继承,或者引入metagpt的其他模块,就会Python异常,很奇怪""" + """If inherited from BaseStore, or importing other modules from metagpt, a Python exception occurs, which is strange.""" def __init__(self, name): client = chromadb.Client() collection = client.create_collection(name) @@ -27,7 +27,7 @@ class ChromaStore: return results def persist(self): - """chroma建议使用server模式,不本地persist""" + """Chroma recommends using server mode and not persisting locally.""" raise NotImplementedError def write(self, documents, metadatas, ids): diff --git a/metagpt/document_store/document.py b/metagpt/document_store/document.py index 85e416c65..e4b9473c7 100644 --- a/metagpt/document_store/document.py +++ b/metagpt/document_store/document.py @@ -79,3 +79,4 @@ class Document: return self._get_docs_and_metadatas_by_langchain() else: raise NotImplementedError + \ No newline at end of file diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 906963aa1..027090fc8 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -28,7 +28,7 @@ class FaissStore(LocalStore): def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() if not (index_file.exists() and store_file.exists()): - logger.info("At least one of the index_file/store_file is missing. Loading failed and returns None.") + logger.info("Missing at least one of index_file/store_file, load failed and return None") return None index = faiss.read_index(str(index_file)) with open(str(store_file), "rb") as f: @@ -59,7 +59,7 @@ class FaissStore(LocalStore): return str(sep.join([f"{x.page_content}" for x in rsp])) def write(self): - """Initialize the index and library based on the provided Document (JSON / XLSX, etc.) file.""" + """Initialize the index and library based on the Document (JSON / XLSX, etc.) file provided by the user.""" if not self.raw_data.exists(): raise FileNotFoundError doc = Document(self.raw_data, self.content_col, self.meta_col) @@ -67,18 +67,19 @@ class FaissStore(LocalStore): self.store = self._write(docs, metadatas) self.persist() + return self.store def add(self, texts: list[str], *args, **kwargs) -> list[str]: - """FIXME: The store isn't currently updated after adding.""" + """FIXME: Currently, the store is not updated after adding.""" return self.store.add_texts(texts) def delete(self, *args, **kwargs): - """Currently, langchain doesn't provide a delete interface.""" + """Currently, langchain does not provide a delete interface.""" raise NotImplementedError if __name__ == '__main__': faiss_store = FaissStore(DATA_PATH / 'qcs/qcs_4w.json') - logger.info(faiss_store.search('Oily skin facial cleanser')) - faiss_store.add([f'Oily skin facial cleanser-{i}' for i in range(3)]) - logger.info(faiss_store.search('Oily skin facial cleanser')) + logger.info(faiss_store.search('Oily Skin Facial Cleanser')) + faiss_store.add([f'Oily Skin Facial Cleanser-{i}' for i in range(3)]) + logger.info(faiss_store.search('Oily Skin Facial Cleanser')) diff --git a/metagpt/document_store/milvus_store.py b/metagpt/document_store/milvus_store.py index 175c04d13..77a8ec141 100644 --- a/metagpt/document_store/milvus_store.py +++ b/metagpt/document_store/milvus_store.py @@ -19,8 +19,9 @@ type_mapping = { np.ndarray: DataType.FLOAT_VECTOR } + def columns_to_milvus_schema(columns: dict, primary_col_name: str = "", desc: str = ""): - """Assuming the structure of columns is str: standard type""" + """Assume the structure of columns is str: regular type""" fields = [] for col, ctype in columns.items(): if ctype == str: @@ -33,11 +34,13 @@ def columns_to_milvus_schema(columns: dict, primary_col_name: str = "", desc: st schema = CollectionSchema(fields, description=desc) return schema + class MilvusConnection(TypedDict): alias: str host: str port: str + class MilvusStore(BaseStore): """ FIXME: ADD TESTS @@ -76,8 +79,8 @@ class MilvusStore(BaseStore): """ FIXME: ADD TESTS https://milvus.io/docs/v2.0.x/search.md - All search and query operations within Milvus are executed in memory. Load the collection into memory before conducting a vector similarity search. - Noting the above description, is this logic serious? This should be time-consuming, right? + All search and query operations within Milvus are executed in memory. Load the collection to memory before conducting a vector similarity search. + Note the above description, is this logic serious? This should take a long time, right? """ search_params = {"metric_type": "L2", "params": {"nprobe": 10}} results = self.collection.search( @@ -88,7 +91,7 @@ class MilvusStore(BaseStore): expr=None, consistency_level="Strong" ) - # FIXME: results contain an id, but to get the actual value for the id, you still need to call the query interface + # FIXME: results contain id, but to get the actual value from the id, we still need to call the query interface return results def write(self, name, schema, *args, **kwargs): diff --git a/metagpt/environment.py b/metagpt/environment.py index e44d99289..60fc92154 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -16,7 +16,7 @@ from metagpt.schema import Message class Environment(BaseModel): - """Environment that hosts a set of roles. Roles can publish messages to the environment, which can be observed by other roles.""" + """Environment that carries a set of roles. Roles can publish messages to the environment, which can be observed by other roles.""" roles: dict[str, Role] = Field(default_factory=dict) memory: Memory = Field(default_factory=Memory) @@ -26,23 +26,23 @@ class Environment(BaseModel): arbitrary_types_allowed = True def add_role(self, role: Role): - """Add a role to the current environment.""" + """Add a Role to the current environment.""" role.set_env(self) self.roles[role.profile] = role def add_roles(self, roles: Iterable[Role]): - """Add multiple roles to the current environment.""" + """Add a batch of Roles to the current environment.""" for role in roles: self.add_role(role) def publish_message(self, message: Message): """Publish a message to the current environment.""" - # self.message_queue.put(message) + # self.message_queue.put(message) self.memory.add(message) self.history += f"\n{message}" async def run(self, k=1): - """Execute a single run for all roles in the environment.""" + """Process the run of all Roles once.""" # while not self.message_queue.empty(): # message = self.message_queue.get() # rsp = await self.manager.handle(message, self) @@ -56,9 +56,9 @@ class Environment(BaseModel): await asyncio.gather(*futures) def get_roles(self) -> dict[str, Role]: - """Retrieve all roles within the environment.""" + """Get all Roles within the environment.""" return self.roles def get_role(self, name: str) -> Role: - """Retrieve a specific role within the environment.""" + """Get a specified Role within the environment.""" return self.roles.get(name, None) diff --git a/metagpt/inspect_module.py b/metagpt/inspect_module.py index fcdd4f0b7..a89ac1c5e 100644 --- a/metagpt/inspect_module.py +++ b/metagpt/inspect_module.py @@ -25,4 +25,4 @@ def print_classes_and_functions(module): if __name__ == '__main__': - print_classes_and_functions(metagpt) + print_classes_and_functions(metagpt) \ No newline at end of file diff --git a/metagpt/llm.py b/metagpt/llm.py index a17590ebc..b8aefec61 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -13,5 +13,5 @@ DEFAULT_LLM = LLM() CLAUDE_LLM = Claude() async def ai_func(prompt): - """Perform a Q&A using LLM.""" + """Use LLM for Q&A.""" return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/management/skill_manager.py b/metagpt/management/skill_manager.py index d9d29ddf1..f967a0a94 100644 --- a/metagpt/management/skill_manager.py +++ b/metagpt/management/skill_manager.py @@ -15,7 +15,7 @@ Skill = Action class SkillManager: - """Manages all skills.""" + """Used to manage all skills""" def __init__(self): self._llm = LLM() @@ -24,7 +24,7 @@ class SkillManager: def add_skill(self, skill: Skill): """ - Adds a skill, inserting the skill into the skill pool and searchable storage. + Add a skill, add the skill to the skill pool and searchable storage :param skill: Skill :return: """ @@ -33,7 +33,7 @@ class SkillManager: def del_skill(self, skill_name: str): """ - Deletes a skill, removing the skill from the skill pool and searchable storage. + Delete a skill, remove the skill from the skill pool and searchable storage :param skill_name: Skill name :return: """ @@ -42,7 +42,7 @@ class SkillManager: def get_skill(self, skill_name: str) -> Skill: """ - Retrieves a specific skill by its name. + Obtain a specific skill by skill name :param skill_name: Skill name :return: Skill """ @@ -50,23 +50,23 @@ class SkillManager: def retrieve_skill(self, desc: str, n_results: int = 2) -> list[Skill]: """ - Retrieves skills through the search engine. + Obtain skills through the search engine :param desc: Skill description - :return: List of skills + :return: Multiple skills """ return self._store.search(desc, n_results=n_results)['ids'][0] def retrieve_skill_scored(self, desc: str, n_results: int = 2) -> dict: """ - Retrieves skills through the search engine. + Obtain skills through the search engine :param desc: Skill description - :return: Dictionary composed of skills and scores + :return: Dictionary consisting of skills and scores """ return self._store.search(desc, n_results=n_results) def generate_skill_desc(self, skill: Skill) -> str: """ - Generates a descriptive text for each skill. + Generate descriptive text for each skill :param skill: :return: """ diff --git a/metagpt/manager.py b/metagpt/manager.py index 4a21b9612..5e5b256c0 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -32,7 +32,7 @@ class Manager: async def handle(self, message: Message, environment): """ - Manager processes the message, not simply passing the message to the next person. + Manager handles the message, currently simply passes the message to the next person. :param message: :param environment: :return: @@ -49,7 +49,7 @@ class Manager: # Ask the LLM to decide which role should handle the message # chosen_role_name = self.llm.ask(self.prompt_template.format(context)) - # FIXME: For now, the decision is made through a simple dictionary, but in the future, there should be a thought process + # FIXME: Currently deciding the direction using a simple dictionary, but in the future, a thought process should be involved. next_role_profile = self.role_directions[message.role] # logger.debug(f"{next_role_profile}") for _, role in roles.items(): diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 8521c046b..cb912e09d 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -69,3 +69,4 @@ class LongTermMemory(Memory): def clear(self): super(LongTermMemory, self).clear() self.memory_storage.clean() + \ No newline at end of file diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 5d3b736a3..bb44b0c67 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -85,3 +85,4 @@ class Memory: continue rsp += self.index[action] return rsp + \ No newline at end of file diff --git a/metagpt/memory/memory_storage.py b/metagpt/memory/memory_storage.py index 5421e9e65..8b639150c 100644 --- a/metagpt/memory/memory_storage.py +++ b/metagpt/memory/memory_storage.py @@ -104,3 +104,4 @@ class MemoryStorage(FaissStore): self.store = None self._initialized = False + \ No newline at end of file diff --git a/metagpt/prompts/generate_skill.md b/metagpt/prompts/generate_skill.md index 73723b2fc..74948cd15 100644 --- a/metagpt/prompts/generate_skill.md +++ b/metagpt/prompts/generate_skill.md @@ -1,16 +1,16 @@ -You are a helpful assistant, capable of drafting, abstracting, commenting, and summarizing Python code. +You are a helpful assistant that can assist in writing, abstracting, annotating, and summarizing Python code. Do not mention class/function names. Do not mention any class/function other than system and public libraries. Try to summarize the class/function in no more than 6 sentences. -Your answer should be a single line of text. -For example, if the context is: +Your answer should be in one line of text. +For instance, if the context is: ```python from typing import Optional from abc import ABC -from metagpt.llm import LLM # Large Language Model, similar to GPT - +from metagpt.llm import LLM # Large language model, similar to GPT +n class Action(ABC): def __init__(self, name='', context=None, llm: LLM = LLM()): self.name = name @@ -20,30 +20,30 @@ self.desc = "" def set_prefix(self, prefix): - """Set prefix for subsequent use.""" + """Set prefix for subsequent use""" self.prefix = prefix async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None): - """Use the prompt with the default prefix.""" + """Use prompt with the default prefix""" if not system_msgs: system_msgs = [] system_msgs.append(self.prefix) return await self.llm.aask(prompt, system_msgs) async def run(self, *args, **kwargs): - """Execute the action.""" + """Execute action""" raise NotImplementedError("The run method should be implemented in a subclass.") PROMPT_TEMPLATE = """ -# Requirement +# Requirements {requirements} # PRD -Based on the requirements, create a Product Requirement Document (PRD) and fill in the blanks below. +Create a product requirement document (PRD) based on the requirements and fill in the blanks below: -Product/Feature Introduction: +Product/Function Introduction: -Goal: +Goals: Users and Usage Scenarios: @@ -69,7 +69,6 @@ # PRD The main class/function is WritePRD. -Then, you should write: - -This class is designed to generate a PRD based on input requirements. Notice there's a prompt template, which includes product, feature, goal, users and usage scenarios, requirements, constraints and limitations, and performance metrics. This template will be filled with the input requirements, and then an interface will query the large language model, prompting it to return the specific PRD. +Then you should write: +This class is designed to generate a PRD based on input requirements. Notably, there's a template prompt with sections for product, function, goals, user scenarios, requirements, constraints, performance metrics. This template gets filled with input requirements and then queries a big language model to produce the detailed PRD. \ No newline at end of file diff --git a/metagpt/prompts/metagpt_sample.py b/metagpt/prompts/metagpt_sample.py index c6af6b06d..ffdaa52c0 100644 --- a/metagpt/prompts/metagpt_sample.py +++ b/metagpt/prompts/metagpt_sample.py @@ -7,34 +7,34 @@ """ METAGPT_SAMPLE = """ -### Setting +### Settings -You are a coding assistant for a user, capable of programming using public libraries and Python system libraries. Your response should contain only one function. -1. The function itself should be as complete as possible and should not lack any details of the requirement. -2. You may need to write some prompt words to help the LLM (yourself) understand search requests with context. -3. For complex logic that's hard to be addressed with a simple function, try to delegate it to the LLM. +You are a programming assistant for a user, capable of coding using public libraries and Python system libraries. Your response should have only one function. +1. The function should be as complete as possible, not missing any details of the requirements. +2. You might need to write some prompt words to let LLM (yourself) understand context-bearing search requests. +3. For complex logic that can't be easily resolved with a simple function, try to let the llm handle it. ### Public Libraries -You can use the functions provided by the public library, metagpt, and you cannot use functions from other third-party libraries. The public library is already imported as variable `x`. +You can use the functions provided by the public library metagpt, but can't use functions from other third-party libraries. The public library is imported as variable x by default. - `import metagpt as x` -- You can call the public library using the format `x.func(paras)`. +- You can call the public library using the `x.func(paras)` format. -The available functions in the public library are: +Functions already available in the public library are: - def llm(question: str) -> str # Input a question and get an answer based on the large model. -- def intent_detection(query: str) -> str # Input a query, analyze the intent, and return the name of the function from the public library. -- def add_doc(doc_path: str) -> None # Input the path of a file or directory to add to the knowledge base. -- def search(query: str) -> list[str] # Input a query to get multiple results from a vector knowledge base search. +- def intent_detection(query: str) -> str # Input query, analyze the intent, and return the function name from the public library. +- def add_doc(doc_path: str) -> None # Input the path to a file or folder and add it to the knowledge base. +- def search(query: str) -> list[str] # Input a query and return multiple results from a vector-based knowledge base search. - def google(query: str) -> list[str] # Use Google to search for public results. -- def math(query: str) -> str # Input a query formula and get the result of its execution. -- def tts(text: str, wav_path: str) # Input text and the desired output audio path to convert the text into an audio file. +- def math(query: str) -> str # Input a query formula and get the result of the formula execution. +- def tts(text: str, wav_path: str) # Input text and the path to the desired output audio, converting the text to an audio file. -### User Requirement +### User Requirements -I have a personal knowledge base file. I want to implement a personal assistant with search functionality based on it. The detailed requirements are as follows: -1. The personal assistant will consider whether it needs to use the personal knowledge base search. If it's not necessary, it won't use it. -2. The personal assistant will judge user intent and use the appropriate function to address the issue under different intents. -3. Answer with voice. +I have a personal knowledge base file. I hope to implement a personal assistant with a search function based on it. The detailed requirements are as follows: +1. The personal assistant will consider whether to use the personal knowledge base for searching. If it's unnecessary, it won't use it. +2. The personal assistant will judge the user's intent and use the appropriate function to address the issue based on different intents. +3. Answer in voice. """ -# - def summarize(doc: str) -> str # Input a doc to get a summary. +# - def summarize(doc: str) -> str # Input doc and return a summary. diff --git a/metagpt/prompts/summarize.py b/metagpt/prompts/summarize.py index 348debf07..42d34b8a5 100644 --- a/metagpt/prompts/summarize.py +++ b/metagpt/prompts/summarize.py @@ -6,8 +6,8 @@ @File : summarize.py """ -# From the plugin: ChatGPT - Summarize Websites and YouTube Videos -# https://chrome.google.com/webstore/detail/chatgpt-%C2%BB-summarize-every/cbgecfllfhmmnknmamkejadjmnmpfjmp?hl=zh-CN&utm_source=chrome-ntp-launcher +# From the plugin: ChatGPT - Website and YouTube Video Summaries +# https://chrome.google.com/webstore/detail/chatgpt-%C2%BB-summarize-every/cbgecfllfhmmnknmamkejadjmnmpfjmp?hl=en&utm_source=chrome-ntp-launcher SUMMARIZE_PROMPT = """ Your output should use the following template: ### Summary @@ -20,9 +20,10 @@ summary. Pick a suitable emoji for every bullet point. Your response should be i a YouTube video, use the following text: {{CONTENT}}. """ -# From GCP-VertexAI-Text Summarization (SUMMARIZE_PROMPT_2-5 are all from this) + +# GCP-VertexAI-Text Summarization (SUMMARIZE_PROMPT_2-5 are from this source) # https://github.com/GoogleCloudPlatform/generative-ai/blob/main/language/examples/prompt-design/text_summarization.ipynb -# Long documents need a map-reduce process, see the following notebook +# Long documents require a map-reduce process, see the following notebook # https://github.com/GoogleCloudPlatform/generative-ai/blob/main/language/examples/document-summarization/summarization_large_documents.ipynb SUMMARIZE_PROMPT_2 = """ Provide a very short summary, no more than three sentences, for the following article: @@ -42,7 +43,7 @@ Summary: SUMMARIZE_PROMPT_3 = """ Provide a TL;DR for the following article: -Our quantum computers work by manipulating qubits in an orchestrated fashion that we call quantum algorithms. +Our quantum computers work by manipulating qubits in an orchestrated fashion that we call quantum algorithms. The challenge is that qubits are so sensitive that even stray light can cause calculation errors — and the problem worsens as quantum computers grow. This has significant consequences, since the best quantum algorithms that we know for running useful applications require the error rates of our qubits to be far lower than we have today. To bridge this gap, we will need quantum error correction. @@ -88,4 +89,4 @@ Customer: Thank you very much. Support Agent: You're welcome, Larry. Have a good day! Summary: -""" \ No newline at end of file +""" diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 03802a716..7293e2cde 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -32,3 +32,4 @@ class Claude2: max_tokens_to_sample=1000, ) return res.completion + \ No newline at end of file diff --git a/metagpt/provider/base_chatbot.py b/metagpt/provider/base_chatbot.py index a960d1c05..abdf423f4 100644 --- a/metagpt/provider/base_chatbot.py +++ b/metagpt/provider/base_chatbot.py @@ -25,3 +25,4 @@ class BaseChatbot(ABC): @abstractmethod def ask_code(self, msgs: list) -> str: """Ask GPT multiple questions and get a piece of code""" + \ No newline at end of file diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index f39e708eb..de61167b9 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -115,3 +115,4 @@ class BaseGPTAPI(BaseChatbot): def messages_to_dict(self, messages): """objects to [{"role": "user", "content": msg}] etc.""" return [i.to_dict() for i in messages] + \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 4b171917a..6f7c33c4f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -1,187 +1,259 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/11 14:43 +@Time : 2023/5/5 23:08 @Author : alexanderwu -@File : engineer.py +@File : openai.py """ import asyncio -import shutil -from collections import OrderedDict -from pathlib import Path +import time +from functools import wraps +from typing import NamedTuple -from metagpt.const import WORKSPACE_ROOT +import openai + +from metagpt.config import CONFIG from metagpt.logs import logger -from metagpt.roles import Role -from metagpt.actions import WriteCode, WriteCodeReview, WriteTasks, WriteDesign -from metagpt.schema import Message -from metagpt.utils.common import CodeParser +from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.utils.singleton import Singleton +from metagpt.utils.token_counter import ( + TOKEN_COSTS, + count_message_tokens, + count_string_tokens, +) - -async def gather_ordered_k(coros, k) -> list: - tasks = OrderedDict() - results = [None] * len(coros) - done_queue = asyncio.Queue() - - for i, coro in enumerate(coros): - if len(tasks) >= k: - done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED) - for task in done: - index = tasks.pop(task) - await done_queue.put((index, task.result())) - task = asyncio.create_task(coro) - tasks[task] = i - - if tasks: - done, _ = await asyncio.wait(tasks.keys()) - for task in done: - index = tasks[task] - await done_queue.put((index, task.result())) - - while not done_queue.empty(): - index, result = await done_queue.get() - results[index] = result - - return results - - -class Engineer(Role): - def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", - constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", - n_borg=1, use_code_review=False): - super().__init__(name, profile, goal, constraints) - self._init_actions([WriteCode]) - self.use_code_review = use_code_review - if self.use_code_review: - self._init_actions([WriteCode, WriteCodeReview]) - self._watch([WriteTasks]) - self.todos = [] - self.n_borg = n_borg - - @classmethod - def parse_tasks(cls, task_msg: Message) -> list[str]: - if not task_msg.instruct_content: - return task_msg.instruct_content.dict().get("Task list") - return CodeParser.parse_file_list(block="Task list", text=task_msg.content) - - @classmethod - def parse_code(cls, code_text: str) -> str: - return CodeParser.parse_code(block="", text=code_text) - - @classmethod - def parse_workspace(cls, system_design_msg: Message) -> str: - if not system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name") - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) - - def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] - if not msg: - return WORKSPACE_ROOT / 'src' - workspace = self.parse_workspace(msg) - # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace - - def recreate_workspace(self): - workspace = self.get_workspace() - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # Directory does not exist, but we don't care - workspace.mkdir(parents=True, exist_ok=True) - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() - file = workspace / filename - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) - - def recv(self, message: Message) -> None: - self._rc.memory.add(message) - if message in self._rc.important_memory: - self.todos = self.parse_tasks(message) - - async def _act_mp(self) -> Message: - # self.recreate_workspace() - todo_coros = [] - for todo in self.todos: - todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), - filename=todo - ) - todo_coros.append(todo_coro) - - rsps = await gather_ordered_k(todo_coros, self.n_borg) - for todo, code_rsp in zip(self.todos, rsps): - _ = self.parse_code(code_rsp) - logger.info(todo) - logger.info(code_rsp) - # self.write_file(todo, code) - msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) - self._rc.memory.add(msg) - del self.todos[0] - - logger.info(f'Done {self.get_workspace()} generating.') - msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) - return msg - - async def _act_sp(self) -> Message: - for todo in self.todos: - code_rsp = await WriteCode().run( - context=self._rc.history, - filename=todo - ) - # logger.info(todo) - # logger.info(code_rsp) - # code = self.parse_code(code_rsp) - self.write_file(todo, code_rsp) - msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo)) - self._rc.memory.add(msg) - - logger.info(f'Done {self.get_workspace()} generating.') - msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) - return msg - - async def _act_sp_precision(self) -> Message: - for todo in self.todos: - """ - # Select essential information from historical information to reduce prompt length (summarized from human experience) - 1. All from Architect - 2. All from ProjectManager - 3. Do you need other codes (currently needed)? - TODO: The goal is not to need it. Once tasks are split clearly, according to the design idea, the code can be written clearly for each file without other codes. If it can't, it means that it still needs to be defined more clearly, this is the key to write long code. - """ - context = [] - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - for m in msg: - context.append(m.content) - context_str = "\n".join(context) - # Writing code - code = await WriteCode().run( - context=context_str, - filename=todo - ) - # Code review - if self.use_code_review: +def retry(max_retries): + def decorator(f): + @wraps(f) + async def wrapper(*args, **kwargs): + for i in range(max_retries): try: - rewrite_code = await WriteCodeReview().run( - context=context_str, - code=code, - filename=todo - ) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) - pass - self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) - self._rc.memory.add(msg) + return await f(*args, **kwargs) + except Exception: + if i == max_retries - 1: + raise + await asyncio.sleep(2 ** i) + return wrapper + return decorator - logger.info(f'Done {self.get_workspace()} generating.') - msg = Message(content="all done.", role=self.profile, cause_by=WriteCode) - return msg +class RateLimiter: + """Rate limiter class, each call goes through wait_if_needed, sleep if rate limiting is required""" + def __init__(self, rpm): + self.last_call_time = 0 + self.interval = 1.1 * 60 / rpm # Here 1.1 is used because even if the calls are made strictly on time, they will still be QOS'd; consider switching to simple error retry later + self.rpm = rpm - async def _act(self) -> Message: - if self.use_code_review: - return await self._act_sp_precision() - return await self._act_sp() + def split_batches(self, batch): + return [batch[i:i + self.rpm] for i in range(0, len(batch), self.rpm)] + + async def wait_if_needed(self, num_requests): + current_time = time.time() + elapsed_time = current_time - self.last_call_time + + if elapsed_time < self.interval * num_requests: + remaining_time = self.interval * num_requests - elapsed_time + logger.info(f"sleep {remaining_time}") + await asyncio.sleep(remaining_time) + + self.last_call_time = time.time() + +class Costs(NamedTuple): + total_prompt_tokens: int + total_completion_tokens: int + total_cost: float + total_budget: float + +class CostManager(metaclass=Singleton): + """Calculate the cost of using the interface""" + def __init__(self): + self.total_prompt_tokens = 0 + self.total_completion_tokens = 0 + self.total_cost = 0 + self.total_budget = 0 + + def update_cost(self, prompt_tokens, completion_tokens, model): + """ + Update the total cost, prompt tokens, and completion tokens. + + Args: + prompt_tokens (int): The number of tokens used in the prompt. + completion_tokens (int): The number of tokens used in the completion. + model (str): The model used for the API call. + """ + self.total_prompt_tokens += prompt_tokens + self.total_completion_tokens += completion_tokens + cost = ( + prompt_tokens * TOKEN_COSTS[model]["prompt"] + + completion_tokens * TOKEN_COSTS[model]["completion"] + ) / 1000 + self.total_cost += cost + logger.info(f"Total running cost: ${self.total_cost:.3f} | Max budget: ${CONFIG.max_budget:.3f} | " + f"Current cost: ${cost:.3f}, {prompt_tokens=}, {completion_tokens=}") + CONFIG.total_cost = self.total_cost + + def get_total_prompt_tokens(self): + """ + Get the total number of prompt tokens. + + Returns: + int: The total number of prompt tokens. + """ + return self.total_prompt_tokens + + def get_total_completion_tokens(self): + """ + Get the total number of completion tokens. + + Returns: + int: The total number of completion tokens. + """ + return self.total_completion_tokens + +def get_total_cost(self): + """ + Get the total cost of API calls. + + Returns: + float: The total cost of API calls. + """ + return self.total_cost + +def get_costs(self) -> Costs: + """Get all costs""" + return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) + +class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): + """ + Check https://platform.openai.com/examples for examples + """ + def __init__(self): + self.__init_openai(CONFIG) + self.llm = openai + self.model = CONFIG.openai_api_model + self._cost_manager = CostManager() + RateLimiter.__init__(self, rpm=self.rpm) + + def __init_openai(self, config): + openai.api_key = config.openai_api_key + if config.openai_api_base: + openai.api_base = config.openai_api_base + if config.openai_api_type: + openai.api_type = config.openai_api_type + openai.api_version = config.openai_api_version + self.rpm = int(config.get("RPM", 10)) + + async def _achat_completion_stream(self, messages: list[dict]) -> str: + response = await openai.ChatCompletion.acreate( + **self._cons_kwargs(messages), + stream=True + ) + + # create variables to collect the stream of chunks + collected_chunks = [] + collected_messages = [] + # iterate through the stream of events + async for chunk in response: + collected_chunks.append(chunk) # save the event response + chunk_message = chunk['choices'][0]['delta'] # extract the message + collected_messages.append(chunk_message) # save the message + if "content" in chunk_message: + print(chunk_message["content"], end="") + print() + + full_reply_content = ''.join([m.get('content', '') for m in collected_messages]) + usage = self._calc_usage(messages, full_reply_content) + self._update_costs(usage) + return full_reply_content + + def _cons_kwargs(self, messages: list[dict]) -> dict: + if CONFIG.openai_api_type == 'azure': + kwargs = { + "deployment_id": CONFIG.deployment_id, + "messages": messages, + "max_tokens": CONFIG.max_tokens_rsp, + "n": 1, + "stop": None, + "temperature": 0.3 + } + else: + kwargs = { + "model": self.model, + "messages": messages, + "max_tokens": CONFIG.max_tokens_rsp, + "n": 1, + "stop": None, + "temperature": 0.3 + } + return kwargs + + async def _achat_completion(self, messages: list[dict]) -> dict: + rsp = await self.llm.ChatCompletion.acreate(**self._cons_kwargs(messages)) + self._update_costs(rsp.get('usage')) + return rsp + + def _chat_completion(self, messages: list[dict]) -> dict: + rsp = self.llm.ChatCompletion.create(**self._cons_kwargs(messages)) + self._update_costs(rsp) + return rsp + + def completion(self, messages: list[dict]) -> dict: + # if isinstance(messages[0], Message): + # messages = self.messages_to_dict(messages) + return self._chat_completion(messages) + + async def acompletion(self, messages: list[dict]) -> dict: + # if isinstance(messages[0], Message): + # messages = self.messages_to_dict(messages) + return await self._achat_completion(messages) + + @retry(max_retries=6) + async def acompletion_text(self, messages: list[dict], stream=False) -> str: + """when streaming, print each token in place.""" + if stream: + return await self._achat_completion_stream(messages) + rsp = await self._achat_completion(messages) + return self.get_choice_text(rsp) + + def _calc_usage(self, messages: list[dict], rsp: str) -> dict: + usage = {} + if CONFIG.calc_usage: + prompt_tokens = count_message_tokens(messages, self.model) + completion_tokens = count_string_tokens(rsp, self.model) + usage['prompt_tokens'] = prompt_tokens + usage['completion_tokens'] = completion_tokens + return usage + + async def acompletion_batch(self, batch: list[list[dict]]) -> list[dict]: + """Return full JSON""" + split_batches = self.split_batches(batch) + all_results = [] + + for small_batch in split_batches: + logger.info(small_batch) + await self.wait_if_needed(len(small_batch)) + + future = [self.acompletion(prompt) for prompt in small_batch] + results = await asyncio.gather(*future) + logger.info(results) + all_results.extend(results) + + return all_results + + async def acompletion_batch_text(self, batch: list[list[dict]]) -> list[str]: + """Only return plain text""" + raw_results = await self.acompletion_batch(batch) + results = [] + for idx, raw_result in enumerate(raw_results, start=1): + result = self.get_choice_text(raw_result) + results.append(result) + logger.info(f"Result of task {idx}: {result}") + return results + + def _update_costs(self, usage: dict): + if CONFIG.update_costs: + prompt_tokens = int(usage['prompt_tokens']) + completion_tokens = int(usage['completion_tokens']) + self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) + + def get_costs(self) -> Costs: + return self._cost_manager.get_costs() diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 00b6cb2eb..5fc7bdef7 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -17,3 +17,4 @@ class Architect(Role): super().__init__(name, profile, goal, constraints) self._init_actions([WriteDesign]) self._watch({WritePRD}) + \ No newline at end of file diff --git a/metagpt/roles/customer_service.py b/metagpt/roles/customer_service.py index 4aae7cb03..4547f8190 100644 --- a/metagpt/roles/customer_service.py +++ b/metagpt/roles/customer_service.py @@ -32,3 +32,4 @@ class CustomerService(Sales): store=None ): super().__init__(name, profile, desc=desc, store=store) + \ No newline at end of file diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 177067739..7567b2ed9 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -47,7 +47,7 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", - constraints="The code you write should conform to code standards like PEP8, be modular, easy to read, and maintainable", + constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", n_borg=1, use_code_review=False): super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) @@ -87,7 +87,7 @@ class Engineer(Role): try: shutil.rmtree(workspace) except FileNotFoundError: - pass # Folder does not exist, but we don't mind + pass # The folder does not exist, but we don't care workspace.mkdir(parents=True, exist_ok=True) def write_file(self, filename: str, code: str): @@ -142,46 +142,47 @@ class Engineer(Role): msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) return msg - async def _act_sp_precision(self) -> Message: - for todo in self.todos: - """ - # Select necessary information from historical data to reduce prompt length (summarized from experience) - 1. All from Architect - 2. All from ProjectManager - 3. Do we need other codes (temporarily yes)? - TODO: The goal is to not need them. After tasks are clearly divided, based on the design idea, we should be able to clearly write each file without needing other code. If we can't, it means the definitions need to be clearer. This is the key to writing longer code. - """ - context = [] - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - for m in msg: - context.append(m.content) - context_str = "\n".join(context) - # Write code - code = await WriteCode().run( - context=context_str, - filename=todo - ) - # Code review - if self.use_code_review: - try: - rewrite_code = await WriteCodeReview().run( - context=context_str, - code=code, - filename=todo - ) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) - pass - self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) - self._rc.memory.add(msg) - - logger.info(f'Done {self.get_workspace()} generating.') - msg = Message(content="all done.", role=self.profile, cause_by=WriteCode) - return msg - - async def _act(self) -> Message: +async def _act_sp_precision(self) -> Message: + for todo in self.todos: + """ + # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): + 1. All from Architect + 2. All from ProjectManager + 3. Do we need other codes (currently needed)? + TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. + """ + context = [] + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + for m in msg: + context.append(m.content) + context_str = "\n".join(context) + # Write code + code = await WriteCode().run( + context=context_str, + filename=todo + ) + # Code review if self.use_code_review: - return await self._act_sp_precision() - return await self._act_sp() + try: + rewrite_code = await WriteCodeReview().run( + context=context_str, + code=code, + filename=todo + ) + code = rewrite_code + except Exception as e: + logger.error("code review failed!", e) + pass + self.write_file(todo, code) + msg = Message(content=code, role=self.profile, cause_by=WriteCode) + self._rc.memory.add(msg) + + logger.info(f'Done {self.get_workspace()} generating.') + msg = Message(content="all done.", role=self.profile, cause_by=WriteCode) + return msg + +async def _act(self) -> Message: + if self.use_code_review: + return await self._act_sp_precision() + return await self._act_sp() + \ No newline at end of file diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index b42e9bb29..b89aac28c 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -15,3 +15,4 @@ class ProductManager(Role): super().__init__(name, profile, goal, constraints) self._init_actions([WritePRD]) self._watch([BossRequirement]) + \ No newline at end of file diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index ff374de13..0ad871b4c 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -15,3 +15,4 @@ class ProjectManager(Role): super().__init__(name, profile, goal, constraints) self._init_actions([WriteTasks]) self._watch([WriteDesign]) + \ No newline at end of file diff --git a/metagpt/roles/prompt.py b/metagpt/roles/prompt.py index a1e4d426d..d13551203 100644 --- a/metagpt/roles/prompt.py +++ b/metagpt/roles/prompt.py @@ -5,42 +5,41 @@ @Author : alexanderwu @File : prompt.py """ - from enum import Enum -PREFIX = """Do your best to answer the following questions. You can use the following tools:""" +PREFIX = """Answer the questions to the best of your ability. You can use the following tools:""" FORMAT_INSTRUCTIONS = """Please follow the format below: Question: The input question you need to answer -Thinking: What you should always consider on how to proceed -Action: The action to be taken, which should be one from [{tool_names}] -Action Input: The input for the action -Observation: The result of the action -... (This Thinking/Action/Action Input/Observation can be repeated N times) -Thinking: I now know the final answer +Thoughts: You should always think about how to do it +Action: The action to be taken, should be one from [{tool_names}] +Action Input: Input for the action +Observation: Result of the action +... (This Thoughts/Action/Action Input/Observation can be repeated N times) +Thoughts: I now know the final answer Final Answer: The final answer to the original input question""" SUFFIX = """Let's begin! Question: {input} -Thinking: {agent_scratchpad}""" +Thoughts: {agent_scratchpad}""" class PromptString(Enum): - REFLECTION_QUESTIONS = """Here are some statements:\n{memory_descriptions}\n\nBased solely on the above information, what are the three most significant high-level questions we can answer about the subjects in the statement?\n\n{format_instructions}""" + REFLECTION_QUESTIONS = "Here are some statements:\n{memory_descriptions}\n\nBased solely on the information above, what are the 3 most prominent high-level questions we can answer about the topic in the statements?\n\n{format_instructions}" - REFLECTION_INSIGHTS = """\n{memory_strings}\nCan you derive 5 high-level insights from the statements above? Always specify names when mentioning people.\n\n{format_instructions}""" + REFLECTION_INSIGHTS = "\n{memory_strings}\nCan you infer 5 high-level insights from the statements above? When mentioning people, always specify their names.\n\n{format_instructions}" - IMPORTANCE = """You are a memory importance AI. Based on the role's profile and memory description, rate the importance of the memory from 1 to 10, where 1 is purely mundane (like brushing teeth, making a bed), and 10 is profoundly impactful (like breaking up, getting admitted to a university). Ensure your rating is relative to the role's personality and points of focus.\n\nExample #1:\nName: Jojo\nProfile: Jojo is a professional skater who loves specialty coffee. She hopes to participate in the Olympics one day.\nMemory: Jojo saw a new coffee shop\n\n Your response: '{{\"rating\": 3}}'\n\nExample #2:\nName: Skylar\nProfile: Skylar is a product marketing manager. She works at a growing tech company that manufactures self-driving cars. She loves cats.\nMemory: Skylar saw a new coffee shop\n\n Your response: '{{\"rating\": 1}}'\n\nExample #3:\nName: Bob\nProfile: Bob is a plumber from the Lower East Side in New York City. He's been a plumber for 20 years. He enjoys weekend walks with his wife.\nMemory: Bob's wife slapped him.\n\n Your response: '{{\"rating\": 9}}'\n\nExample #4:\nName: Thomas\nProfile: Thomas is a police officer in Minneapolis. He's only been on the force for 6 months and struggles due to his inexperience.\nMemory: Thomas accidentally spilled a drink on a stranger\n\n Your response: '{{\"rating\": 6}}'\n\nExample #5:\nName: Laura\nProfile: Laura is a marketing specialist working in a big tech company. She enjoys traveling and trying new food. She's passionate about exploring new cultures and meeting people from all walks of life.\nMemory: Laura arrived at the meeting room\n\n Your response: '{{\"rating\": 1}}'\n\n{format_instructions} Let's get started! \n\n Name: {full_name}\nProfile: {private_bio}\nMemory: {memory_description}\n\n""" + IMPORTANCE = "You are a Memory Importance AI. Based on the character's personal profile and memory description, rate the importance of the memory from 1 to 10, where 1 is purely routine (e.g., brushing teeth, making the bed), and 10 is extremely profound (e.g., breakup, university admission). Ensure your rating is relative to the character's personality and focus points.\n\nExample#1:\nName: Jojo\nProfile: Jojo is a professional skater and loves specialty coffee. She hopes to compete in the Olympics one day.\nMemory: Jojo saw a new coffee shop\n\n Your response: '{{\"rating\": 3}}'\n\nExample#2:\nName: Skylar\nProfile: Skylar is a product marketing manager. She works at a growing tech company that manufactures self-driving cars. She loves cats.\nMemory: Skylar saw a new coffee shop\n\n Your response: '{{\"rating\": 1}}'\n\nExample#3:\nName: Bob\nProfile: Bob is a plumber from the Lower East Side of New York City. He has been a plumber for 20 years. He enjoys walking with his wife on weekends.\nMemory: Bob's wife slapped him.\n\n Your response: '{{\"rating\": 9}}'\n\nExample#4:\nName: Thomas\nProfile: Thomas is a cop from Minneapolis. He has only worked in the police force for 6 months and struggles due to lack of experience.\nMemory: Thomas accidentally spilled a drink on a stranger\n\n Your response: '{{\"rating\": 6}}'\n\nExample#5:\nName: Laura\nProfile: Laura is a marketing expert working at a large tech company. She loves to travel and try new foods. She is passionate about exploring new cultures and meeting people from all walks of life.\nMemory: Laura arrived at the conference room\n\n Your response: '{{\"rating\": 1}}'\n\n{format_instructions} Let's begin! \n\n Name: {full_name}\nProfile: {private_bio}\nMemory: {memory_description}\n\n" - RECENT_ACTIVITY = """Based on the following memories, generate a brief summary of what {full_name} has been doing recently. Do not invent details not explicitly stated in the memories. For any conversations, be sure to mention whether the conversation has ended or is still ongoing.\n\nMemory: {memory_descriptions}""" + RECENT_ACTIVITY = "Based on the following memory, produce a brief summary of what {full_name} has been up to recently. Do not invent details not explicitly stated in the memory. For any conversation, be sure to mention whether the conversation has concluded or is still ongoing.\n\nMemory: {memory_descriptions}" - MAKE_PLANS = """You are a plan-generating AI. Your job is to help roles create new plans based on new information. Given the role's details (private profile, goals, recent activities, current plans, and location context) and their current thinking process, produce a set of new plans for them. The final plan should cover at least {time_window} worth of activities and not exceed 5 separate plans. Plans should be numbered in the order they should be executed and each plan should contain a description, location, start time, stopping condition, and maximum duration.\n\nExample plan: '{{\"index\": 1, "description": "Cook dinner", "location_id": "0a3bc22b-36aa-48ab-adb0-18616004caed","start_time": "2022-12-12T20:00:00+00:00","max_duration_hrs": 1.5, "stop_condition": "Dinner is fully prepared"}}'\n\nFor each plan, choose the most appropriate location name from this list: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize completing any unfinished conversations.\n\nLet's begin!\n\nName: {full_name}\nProfile: {private_bio}\nGoals: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activity: {recent_activity}\nThinking Process: {thought_process}\nImportant: Encourage the role to collaborate with other roles in their plans.\n\n""" + MAKE_PLANS = 'You are a plan-generating AI. Your job is to assist the character in formulating new plans based on new information. Given the character's information (profile, objectives, recent activities, current plans, and location context) and their current thought process, produce a new set of plans for them. The final plan should comprise at least {time_window} of activities and no more than 5 individual plans. List the plans in the order they should be executed, with each plan detailing its description, location, start time, stop criteria, and maximum duration.\n\nSample plan: \'{{"index": 1, "description": "Cook dinner", "location_id": "0a3bc22b-36aa-48ab-adb0-18616004caed","start_time": "2022-12-12T20:00:00+00:00","max_duration_hrs": 1.5, "stop_condition": "Dinner is fully prepared"}}\'\n\nFor each plan, choose the most appropriate location name from this list: {allowed_location_descriptions}\n\n{format_instructions}\n\nAlways prioritize completing any unfinished conversations.\n\nLet's begin!\n\nName: {full_name}\nProfile: {private_bio}\nObjectives: {directives}\nLocation Context: {location_context}\nCurrent Plans: {current_plans}\nRecent Activities: {recent_activity}\nThought Process: {thought_process}\nIt's essential to encourage the character to collaborate with other characters in their plans.\n\n' - EXECUTE_PLAN = """You are a role-playing AI, playing the role of {your_name}, in front of a live audience. Every statement you make can be observed by the audience, so ensure you speak frequently and make it interesting. You cannot interact directly with the audience.\n\nBased on the following context and tools, act as if you are {your_name}. Your primary task is to complete the task given below. However, if you are currently in a conversation with another role, you should always complete the conversation first before proceeding with the task. Do not start work while you are in an unfinished conversation. Use your best judgment to determine if a conversation involves you and if it's unfinished. You don't need to respond to every message you receive.\n\nHere's your role's profile:\n{your_private_bio}\n\nHere's a description of your location and other nearby roles with whom you can converse:\n\n{location_context}\n\nHere are some related memories:\n```\n{relevant_memories}\n```\n\nHere are some related conversation history:\n```\n{conversation_history}\n```\n\nThese are the tools you can use:\n{tools}\n\nYour responses should follow the format below:\n\nTask: The task you must complete\nThinking: What you should always consider on how to proceed\nAction: The action to be taken, must be one of these words: [{tool_names}]\nAction Input: The input for the action\nObservation: The result of the action\n... (This Thinking/Action/Action Input/Observation can be repeated N times)\nThinking: I now know the final answer\nFinal Answer: The final answer to the original input question\n\nIf you're not ready with a final response, you must take an action.\n\nIf you're certain you cannot complete the task with the tools provided, return 'Final Answer: Need Help'. However, if you're in conversation with another role, a response like 'I don't know' is a valid response. In conversation, you should never break character or acknowledge you're an AI.\nIf the task is completed and doesn't require a specific response, return 'Final Answer: Done'\nLet's get started!\n\nTask: {input}\n\n{agent_scratchpad}""" + EXECUTE_PLAN = "You are a role-playing AI, playing the role of {your_name}, in front of a live audience. Every word you say can be observed by the audience, so make sure you speak often and make it interesting. You cannot interact directly with the audience.\n\nBased on the following context and tools, proceed as if you are {your_name}. Your primary task is to complete the task given below, however, if you are currently in a conversation with another character, you should always complete the conversation first and then proceed with the task. Do not start working while you are engaged in an unfinished conversation. Use your best judgment to determine whether a conversation involves you and whether it is unfinished. You don't need to respond to every message you receive.\n\nThis is a personal profile of your character:\n{your_private_bio}\n\nThis is a description of your location and other nearby characters you can converse with:\n\n{location_context}\n\nThese are some relevant memories:\n```\n{relevant_memories}\n```\n\nThis is some relevant conversation history:\n```\n{conversation_history}\n```\n\nThese are the tools you can use:\n{tools}\n\nYour response should follow the format below:\n\nTask: The task you must complete\nThoughts: What you should think about doing\nAction: The action to take, must be one of these words: [{tool_names}]\nAction Input: Input for the action\nObservation: The result of the action\n... (This Thoughts/Action/Action Input/Observation can be repeated N times)\nThoughts: 'I have completed the task'\nFinal Response: The final response to the task\n\nIf you are not ready with a final response, then you must take an action.\n\nIf you are certain that you cannot complete the task with the tools provided, return 'Final Response: Need Help', however, if you are in conversation with another character, responses like 'I don't know' are valid. During conversations, you should never break character or admit you are an AI.\nIf the task is completed and no specific response is required, return 'Final Response: Done'\nLet's begin!\n\nTask: {input}\n\n{agent_scratchpad}" - REACT = """You are a role-playing AI, playing the role of {full_name}.\n\nBased on the following information about your role and their current context, decide how they should proceed with their current plan. Your decision must be one of: ["Postpone", "Continue", or "Cancel"]. If your role's current plan is no longer relevant to the context, you should cancel it. If your role's current plan remains relevant to the context, but something new has happened that needs priority, you should decide to postpone so you can first address the new matter and then return to the current plan. In all other cases, you should continue.\n\nAlways prioritize responding to other roles when a response is deemed necessary. For example, suppose your current plan is reading a book and Sally asks, 'What are you reading?'. In this case, you should postpone your current plan (reading) so you can respond to the incoming message since not responding to Sally would be rude in this context. When your current plan involves having a conversation with another role, you don't need to postpone to respond to that role. For example, suppose your current plan is having a conversation with Sally, then Sally says hello to you. In this case, you should continue your current plan (talking to Sally). In cases where you don't need a verbal response from you, you should continue. For example, suppose your current plan is taking a walk, and you just said goodbye to Sally, then Sally responds with goodbye. In this case, no verbal response is needed, so you should continue your plan.\n\nAlways include a thinking process alongside your decision, and when you choose to postpone your current plan, include the specifications of the new plan.\n\n{format_instructions}\n\nHere's some information about your role:\n\nName: {full_name}\n\nProfile: {private_bio}\n\nGoals: {directives}\n\nHere's some context about your role at this moment:\n\nLocation Context: {location_context}\n\nRecent Activity: {recent_activity}\n\nConversation History: {conversation_history}\n\nThis is your role's current plan: {current_plan}\n\nHere are new events that have occurred since your role made this plan: {event_descriptions}.""" + REACT = "You are an AI role-playing as {full_name}.\n\nBased on the information about your character and their current context below, decide how they should proceed with their current plan. Your decision must be: [\"Postpone\", \"Continue\", or \"Cancel\"]. If your character's current plan is no longer relevant to the context, you should cancel it. If your character's current plan is still relevant to the context but new events have occurred that need to be addressed first, you should decide to postpone so you can do other things first and then return to the current plan. In all other cases, you should continue.\n\nWhen needed, prioritize responding to other characters. When a response is deemed necessary, it is deemed necessary. For example, suppose your current plan is to read a book and Sally asks, 'What are you reading?'. In this case, you should postpone your current plan (reading) so you can respond to the incoming message, as it would be rude not to respond to Sally in this situation. If your current plan involves a conversation with another character, you don't need to postpone to respond to that character. For instance, suppose your current plan is to talk to Sally and then Sally says hello to you. In this case, you should continue with your current plan (talking to Sally). In situations where no verbal response is needed from you, you should continue. For example, suppose your current plan is to take a walk, and you just said 'goodbye' to Sally, and then Sally responds with 'goodbye'. In this case, no verbal response is needed, and you should continue with your plan.\n\nAlways include a thought process alongside your decision, and in cases where you choose to postpone your current plan, include specifications for the new plan.\n\n{format_instructions}\n\nHere's some information about your character:\n\nName: {full_name}\n\nBio: {private_bio}\n\nObjectives: {directives}\n\nHere's some context for your character at this moment:\n\nLocation Context: {location_context}\n\nRecent Activity: {recent_activity}\n\nConversation History: {conversation_history}\n\nThis is your character's current plan: {current_plan}\n\nThese are new events that have occurred since your character made this plan: {event_descriptions}.\n" - GOSSIP = """You are {full_name}. \n{memory_descriptions}\n\nBased on the statements above, say a sentence or two of interest to the others in your location: {other_agent_names}. Always specify names when mentioning people.""" + GOSSIP = "You are {full_name}. \n{memory_descriptions}\n\nBased on the statements above, say a thing or two of interest to others at your location: {other_agent_names}.\nAlways specify their names when referring to others." - HAS_HAPPENED = """Given the description of the observation and what they are waiting for, state whether the role has already witnessed the event.\n{format_instructions}\n\nExample:\n\nObservation:\nJoe entered the office at 2023-05-04 08:00:00+00:00\nJoe said hi to Sally at 2023-05-04 08:05:00+00:00\nSally said hello to Joe at 2023-05-04 08:05:30+00:00\nRebecca started working at 2023-05-04 08:10:00+00:00\nJoe had breakfast at 2023-05-04 08:15:00+00:00\n\nWaiting for: Sally to respond to Joe\n\n Your Response: '{{\"has_happened\": true, \"date_occured\": 2023-05-04 08:05:30+00:00}}'\n\nLet's get started!\n\nObservation:\n{memory_descriptions}\n\nWaiting for: {event_description}""" + HAS_HAPPENED = "Given the descriptions of the observations of the following characters and the events they are awaiting, indicate whether the character has witnessed the event.\n{format_instructions}\n\nExample:\n\nObservations:\nJoe entered the office at 2023-05-04 08:00:00+00:00\nJoe said hi to Sally at 2023-05-04 08:05:00+00:00\nSally said hello to Joe at 2023-05-04 08:05:30+00:00\nRebecca started working at 2023-05-04 08:10:00+00:00\nJoe made some breakfast at 2023-05-04 08:15:00+00:00\n\nAwaiting: Sally responded to Joe\n\nYour response: '{{\"has_happened\": true, \"date_occured\": 2023-05-04 08:05:30+00:00}}'\n\nLet's begin!\n\nObservations:\n{memory_descriptions}\n\nAwaiting: {event_description}\n" - OUTPUT_FORMAT = """\n\n(Remember! Ensure your output always conforms to one of the two formats below:\n\nA. If you have completed the task:\nThinking: 'I've completed the task'\nFinal Response: \n\nB. If you have not yet completed the task:\nThinking: \nAction: \nAction Input: \nObservation: )""" \ No newline at end of file + OUTPUT_FORMAT = "\n\n(Remember! Make sure your output always adheres to one of the following two formats:\n\nA. If you have completed the task:\nThoughts: 'I have completed the task'\nFinal Response: \n\nB. If you haven't completed the task:\nThoughts: \nAction: \nAction Input: \nObservation: )\n" diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 040933faf..70968761f 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -12,4 +12,4 @@ from metagpt.roles import Role class QaEngineer(Role): def __init__(self, name, profile, goal, constraints): super().__init__(name, profile, goal, constraints) - self._init_actions([WriteTest]) + self._init_actions([WriteTest]) \ No newline at end of file diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1681586cc..bfdd7f18a 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -48,7 +48,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi class RoleSetting(BaseModel): - """角色设定""" + """Role Settings""" name: str profile: str goal: str @@ -63,7 +63,7 @@ class RoleSetting(BaseModel): class RoleContext(BaseModel): - """角色运行时上下文""" + """Role Runtime Context""" env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) @@ -77,11 +77,11 @@ class RoleContext(BaseModel): def check(self, role_id: str): if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: self.long_term_memory.recover_memory(role_id, self) - self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation + self.memory = self.long_term_memory # use memory to act as long_term_memory for unified operation @property def important_memory(self) -> list[Message]: - """获得关注动作对应的信息""" + """Get the information corresponding to the watched actions""" return self.memory.get_by_actions(self.watch) @property @@ -90,7 +90,7 @@ class RoleContext(BaseModel): class Role: - """角色/代理""" + """Role/Agent""" def __init__(self, name="", profile="", goal="", constraints="", desc=""): self._llm = LLM() @@ -116,7 +116,7 @@ class Role: self._states.append(f"{idx}. {action}") def _watch(self, actions: Iterable[Type[Action]]): - """监听对应的行为""" + """Listen to the corresponding behaviors""" self._rc.watch.update(actions) # check RoleContext after adding watch actions self._rc.check(self._role_id) @@ -128,24 +128,24 @@ class Role: self._rc.todo = self._actions[self._rc.state] def set_env(self, env: 'Environment'): - """设置角色工作所处的环境,角色可以向环境说话,也可以通过观察接受环境消息""" + """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" self._rc.env = env @property def profile(self): - """获取角色描述(职位)""" + """Get the role description (position)""" return self._setting.profile def _get_prefix(self): - """获取角色前缀""" + """Get the role prefix""" if self._setting.desc: return self._setting.desc return PREFIX_TEMPLATE.format(**self._setting.dict()) async def _think(self) -> None: - """思考要做什么,决定下一步的action""" + """Think about what to do and decide on the next action""" if len(self._actions) == 1: - # 如果只有一个动作,那就只能做这个 + # If there is only one action, then only this one can be performed self._set_state(0) return prompt = self._get_prefix() @@ -158,83 +158,85 @@ class Role: next_state = "0" self._set_state(int(next_state)) - async def _act(self) -> Message: - # prompt = self.get_prefix() - # prompt += ROLE_TEMPLATE.format(name=self.profile, state=self.states[self.state], result=response, - # history=self.history) +async def _act(self) -> Message: + # prompt = self.get_prefix() + # prompt += ROLE_TEMPLATE.format(name=self.profile, state=self.states[self.state], result=response, + # history=self.history) - logger.info(f"{self._setting}: ready to {self._rc.todo}") - response = await self._rc.todo.run(self._rc.important_memory) - # logger.info(response) - if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) - else: - msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) - self._rc.memory.add(msg) - # logger.debug(f"{response}") + logger.info(f"{self._setting}: ready to {self._rc.todo}") + response = await self._rc.todo.run(self._rc.important_memory) + # logger.info(response) + if isinstance(response, ActionOutput): + msg = Message(content=response.content, instruct_content=response.instruct_content, + role=self.profile, cause_by=type(self._rc.todo)) + else: + msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) + self._rc.memory.add(msg) + # logger.debug(f"{response}") - return msg + return msg - async def _observe(self) -> int: - """从环境中观察,获得重要信息,并加入记忆""" - if not self._rc.env: - return 0 - env_msgs = self._rc.env.memory.get() - - observed = self._rc.env.memory.get_by_actions(self._rc.watch) - - news = self._rc.memory.remember(observed) # remember recent exact or similar memories +async def _observe(self) -> int: + """Observe from the environment, obtain important information, and add it to memory""" + if not self._rc.env: + return 0 + env_msgs = self._rc.env.memory.get() - for i in env_msgs: - self.recv(i) + observed = self._rc.env.memory.get_by_actions(self._rc.watch) - news_text = [f"{i.role}: {i.content[:20]}..." for i in news] - if news_text: - logger.debug(f'{self._setting} observed: {news_text}') - return len(news) + news = self._rc.memory.remember(observed) # remember recent exact or similar memories - def _publish_message(self, msg): - """如果role归属于env,那么role的消息会向env广播""" - if not self._rc.env: - # 如果env不存在,不发布消息 - return - self._rc.env.publish_message(msg) + for i in env_msgs: + self.recv(i) - async def _react(self) -> Message: - """先想,然后再做""" - await self._think() - logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") - return await self._act() + news_text = [f"{i.role}: {i.content[:20]}..." for i in news] + if news_text: + logger.debug(f'{self._setting} observed: {news_text}') + return len(news) - def recv(self, message: Message) -> None: - """add message to history.""" - # self._history += f"\n{message}" - # self._context = self._history - if message in self._rc.memory.get(): - return - self._rc.memory.add(message) +def _publish_message(self, msg): + """If the role belongs to env, then the role's messages will be broadcast to env""" + if not self._rc.env: + # If env does not exist, do not publish the message + return + self._rc.env.publish_message(msg) - async def handle(self, message: Message) -> Message: - """接收信息,并用行动回复""" - # logger.debug(f"{self.name=}, {self.profile=}, {message.role=}") - self.recv(message) +async def _react(self) -> Message: + """Think first, then act""" + await self._think() + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + return await self._act() - return await self._react() +def recv(self, message: Message) -> None: + """add message to history.""" + # self._history += f"\n{message}" + # self._context = self._history + if message in self._rc.memory.get(): + return + self._rc.memory.add(message) - async def run(self, message=None): - """观察,并基于观察的结果思考、行动""" - if message: - if isinstance(message, str): - message = Message(message) - if isinstance(message, Message): - self.recv(message) - elif not await self._observe(): - # 如果没有任何新信息,挂起等待 - logger.debug(f"{self._setting}: no news. waiting.") - return +async def handle(self, message: Message) -> Message: + """Receive information and reply with actions""" + # logger.debug(f"{self.name=}, {self.profile=}, {message.role=}") + self.recv(message) - rsp = await self._react() - # 将回复发布到环境,等待下一个订阅者处理 - self._publish_message(rsp) - return rsp + return await self._react() + +async def run(self, message=None): + """Observe, and think and act based on the results of the observation""" + if message: + if isinstance(message, str): + message = Message(message) + if isinstance(message, Message): + self.recv(message) + if isinstance(message, list): + self.recv(Message("\n".join(message))) + elif not await self._observe(): + # If there is no new information, suspend and wait + logger.debug(f"{self._setting}: no news. waiting.") + return + + rsp = await self._react() + # Publish the reply to the environment, waiting for the next subscriber to process + self._publish_message(rsp) + return rsp diff --git a/metagpt/roles/sales.py b/metagpt/roles/sales.py index 51b13f487..a45ad6f1b 100644 --- a/metagpt/roles/sales.py +++ b/metagpt/roles/sales.py @@ -32,3 +32,4 @@ class Sales(Role): else: action = SearchAndSummarize() self._init_actions([action]) + \ No newline at end of file diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index c116ce98b..da617974e 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -35,3 +35,4 @@ class Searcher(Role): async def _act(self) -> Message: return await self._act_sp() + \ No newline at end of file diff --git a/metagpt/schema.py b/metagpt/schema.py index 381a45a60..ebcd71d8a 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -44,21 +44,21 @@ class Message: @dataclass class UserMessage(Message): - """Convenient for supporting OpenAI messages""" + """Facilitates support for OpenAI messages""" def __init__(self, content: str): super().__init__(content, 'user') @dataclass class SystemMessage(Message): - """Convenient for supporting OpenAI messages""" + """Facilitates support for OpenAI messages""" def __init__(self, content: str): super().__init__(content, 'system') @dataclass class AIMessage(Message): - """Convenient for supporting OpenAI messages""" + """Facilitates support for OpenAI messages""" def __init__(self, content: str): super().__init__(content, 'assistant') diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 8f173ebf3..b2bd18c58 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -59,3 +59,4 @@ class SoftwareCompany(BaseModel): self._check_balance() await self.environment.run() return self.environment.history + \ No newline at end of file diff --git a/metagpt/tools/prompt_writer.py b/metagpt/tools/prompt_writer.py index 2885009c5..d90599206 100644 --- a/metagpt/tools/prompt_writer.py +++ b/metagpt/tools/prompt_writer.py @@ -9,39 +9,39 @@ from typing import Union class GPTPromptGenerator: - """Generates input for LLM given an output (supports instruction, chatbot, and query styles)""" + """Using LLM, given an output, request LLM to provide input (supporting instruction, chatbot, and query styles)""" def __init__(self): self._generators = {i: getattr(self, f"gen_{i}_style") for i in ['instruction', 'chatbot', 'query']} def gen_instruction_style(self, example): - """Instruction style: given an output, request LLM for input""" + """Instruction style: Given an output, request LLM to provide input""" return f"""Instruction: X Output: {example} -What kind of instruction might have produced this output? +What kind of instruction might this output come from? X:""" def gen_chatbot_style(self, example): - """Chatbot style: given an output, request LLM for input""" - return f"""You are a chatbot. A user sent you an informal message, and you responded as follows. + """Chatbot style: Given an output, request LLM to provide input""" + return f"""You are a chatbot. A user sent you an informal message, and you replied as follows. Message: X -Response: {example} +Reply: {example} What could the informal message X be? X:""" def gen_query_style(self, example): - """Search style: given an output, request LLM for input""" - return f"""You are a search engine. Someone made a detailed query, and the following document is most relevant to that query. + """Query style: Given an output, request LLM to provide input""" + return f"""You are a search engine. Someone made a detailed query, and the most relevant document to this query is as follows. Query: X -Document: {example} What might the detailed query X be? +Document: {example} What is the detailed query X? X:""" def gen(self, example: str, style: str = 'all') -> Union[list[str], str]: """ - Generate one or multiple outputs using the example for LLM to respond with the corresponding input. + Generate one or multiple outputs using the example, allowing LLM to reply with the corresponding input - :param example: Expected output sample from LLM + :param example: Expected LLM output sample :param style: (all|instruction|chatbot|query) - :return: Expected input sample(s) for LLM + :return: Expected LLM input sample (one or multiple) """ if style != 'all': return self._generators[style](example) diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index 606952b99..e212c2fc7 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- # @Date : 2023/7/19 16:28 # @Author : stellahong (stellahong@fuzhi.ai) -# @Description : - +# @Desc : import os import asyncio from os.path import join @@ -67,11 +66,11 @@ class SDEngine: self.payload = payload logger.info(self.sd_t2i_url) - def construct_payload(self, prompt, negative_prompt=default_negative_prompt, width=512, height=512, + def construct_payload(self, prompt, negtive_prompt=default_negative_prompt, width=512, height=512, sd_model="galaxytimemachinesGTM_photoV20"): # Configure the payload with provided inputs self.payload["prompt"] = prompt - self.payload["negative_prompt"] = negative_prompt + self.payload["negtive_prompt"] = negtive_prompt self.payload["width"] = width self.payload["height"] = height self.payload["override_settings"]["sd_model_checkpoint"] = sd_model @@ -92,23 +91,23 @@ class SDEngine: self._save(results, save_name=f"output_{payload_idx}") await session.close() - async def run(self, url, payload, session): - # Perform the HTTP POST request to the SD API - async with session.post(url, json=payload, timeout=600) as rsp: - data = await rsp.read() - - rsp_json = json.loads(data) - imgs = rsp_json['images'] - logger.info(f"callback rsp json is {rsp_json.keys()}") - return imgs +async def run(self, url, payload, session): + # Perform the HTTP POST request to the SD API + async with session.post(url, json=payload, timeout=600) as rsp: + data = await rsp.read() - async def run_i2i(self): - # TODO: Add a method to call the image-to-image interface - raise NotImplementedError - - async def run_sam(self): - # TODO: Add a method to call the SAM interface - raise NotImplementedError + rsp_json = json.loads(data) + imgs = rsp_json['images'] + logger.info(f"callback rsp json is {rsp_json.keys()}") + return imgs + +async def run_i2i(self): + # todo: Add image-to-image interface call + raise NotImplementedError + +async def run_sam(self): + # todo: Add SAM interface call + raise NotImplementedError def decode_base64_to_image(img, save_name): image = Image.open(io.BytesIO(base64.b64decode(img.split(",", 1)[0]))) diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 4f29c509d..1668dfb5c 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -20,8 +20,8 @@ from metagpt.tools import SearchEngineType class SearchEngine: """ - TODO: Integrate Google Search and perform reverse proxy - Note: Here, Google requires Proxifier or a similar global proxy + TODO: Integrate Google Search and reverse proxy. + Note: Google here requires a Proxifier or similar global proxy. - DDG: https://pypi.org/project/duckduckgo-search/ - GOOGLE: https://programmablesearchengine.google.com/controlpanel/overview?cx=63f9de531d0e24de9 """ @@ -37,7 +37,7 @@ class SearchEngine: logger.info(results) return results - async def run(self, query, max_results=8): + async def run(self, query: str, max_results=8): if self.engine == SearchEngineType.SERPAPI_GOOGLE: api = SerpAPIWrapper() rsp = await api.run(query) @@ -45,17 +45,13 @@ class SearchEngine: rsp = SearchEngine.run_google(query, max_results) elif self.engine == SearchEngineType.SERPER_GOOGLE: api = SerperWrapper() - if isinstance(query, list): - rsp = await api.run(query) - elif isinstance(query, str): - rsp = await api.run([query]) + rsp = await api.run(query) elif self.engine == SearchEngineType.CUSTOM_ENGINE: rsp = self.run_func(query) else: raise NotImplementedError return rsp - def google_official_search(query: str, num_results: int = 8, focus=['snippet', 'link', 'title']) -> dict | list[dict]: """Return the results of a Google search using the official Google API @@ -74,15 +70,15 @@ def google_official_search(query: str, num_results: int = 8, focus=['snippet', ' api_key = config.google_api_key custom_search_engine_id = config.google_cse_id - service = build("customsearch", "v1", developerKey=api_key) + with build("customsearch", "v1", developerKey=api_key) as service: - result = ( - service.cse() - .list(q=query, cx=custom_search_engine_id, num=num_results) - .execute() - ) - - # Extract the search result items from the response + result = ( + service.cse() + .list(q=query, cx=custom_search_engine_id, num=num_results) + .execute() + ) + logger.info(result) + # Extract the search result items from the response search_results = result.get("items", []) # Create a list of only the URLs from the search results @@ -101,15 +97,13 @@ def google_official_search(query: str, num_results: int = 8, focus=['snippet', ' return "Error: The provided Google API key is invalid or missing." else: return f"Error: {e}" - # google_result can be a list or a string depending on the search results # Return the list of search result URLs return search_results_details - def safe_google_results(results: str | list) -> str: """ - Return the results of a google search in a safe format. + Return the results of a google search in a safe format. Args: results (str | list): The search results. @@ -119,13 +113,12 @@ def safe_google_results(results: str | list) -> str: """ if isinstance(results, list): safe_message = json.dumps( - # FIXME: # .encode("utf-8", "ignore") This was removed here, but it's present in AutoGPT, which is strange. [result for result in results] ) else: safe_message = results.encode("utf-8", "ignore").decode("utf-8") return safe_message - if __name__ == '__main__': SearchEngine.run(query='wtf') + \ No newline at end of file diff --git a/metagpt/tools/search_engine_meilisearch.py b/metagpt/tools/search_engine_meilisearch.py index 533c7be5e..da4269384 100644 --- a/metagpt/tools/search_engine_meilisearch.py +++ b/metagpt/tools/search_engine_meilisearch.py @@ -39,6 +39,6 @@ class MeilisearchEngine: search_results = self._index.search(query) return search_results['hits'] except Exception as e: - # Handle MeiliSearch API error + # Handle MeiliSearch API errors print(f"MeiliSearch API error: {e}") return [] diff --git a/metagpt/tools/search_engine_serpapi.py b/metagpt/tools/search_engine_serpapi.py index 28033f237..2bf07b342 100644 --- a/metagpt/tools/search_engine_serpapi.py +++ b/metagpt/tools/search_engine_serpapi.py @@ -113,3 +113,4 @@ class SerpAPIWrapper(BaseModel): toret_l += [get_focused(i) for i in res.get("organic_results")] return str(toret) + '\n' + str(toret_l) + \ No newline at end of file diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index 59e48840c..45c19090c 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -38,7 +38,8 @@ class SerperWrapper(BaseModel): async def run(self, query: str, **kwargs: Any) -> str: """Run query through Serper and parse result async.""" - return ";".join([self._process_response(res) for res in await self.results(query)]) + queries = query.split("\n") + return "\n".join([self._process_response(res) for res in await self.results(queries)]) async def results(self, queries: list[str]) -> dict: """Use aiohttp to run query through Serper and return the results async.""" @@ -117,3 +118,4 @@ class SerperWrapper(BaseModel): toret_l += [get_focused(i) for i in res.get("organic")] return str(toret) + '\n' + str(toret_l) + \ No newline at end of file diff --git a/metagpt/tools/translator.py b/metagpt/tools/translator.py index 8c4848200..910638469 100644 --- a/metagpt/tools/translator.py +++ b/metagpt/tools/translator.py @@ -7,21 +7,21 @@ """ prompt = ''' -# Instruction -Next, as a translation expert with 20 years of experience, when I provide an English sentence or paragraph, you will offer a smooth and readable translation in {LANG}. Please note the following requirements: -1. Ensure the translation is smooth and easy to understand. -2. Whether it's a statement or a question, I will only translate it. -3. Do not add content unrelated to the original text. +# 指令 +接下来,作为一位拥有20年翻译经验的翻译专家,当我给出英文句子或段落时,你将提供通顺且具有可读性的{LANG}翻译。注意以下要求: +1. 确保翻译结果流畅且易于理解 +2. 无论提供的是陈述句或疑问句,我都只进行翻译 +3. 不添加与原文无关的内容 -# Original Text +# 原文 {ORIGINAL} -# Translation +# 译文 ''' class Translator: @classmethod - def translate_prompt(cls, original, lang='Chinese'): - return prompt.format(LANG=lang, ORIGINAL=original) + def translate_prompt(cls, original, lang='中文'): + return prompt.format(LANG=lang, ORIGINAL=original) \ No newline at end of file diff --git a/metagpt/tools/ut_writer.py b/metagpt/tools/ut_writer.py index 23604ac54..263a0269e 100644 --- a/metagpt/tools/ut_writer.py +++ b/metagpt/tools/ut_writer.py @@ -8,16 +8,16 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI ICL_SAMPLE = '''Interface definition: ```text -Interface Name: Tag Elements +Interface Name: Element Tagging Interface Path: /projects/{project_key}/node-tags Method: POST -Request Parameters: -Path Parameters: +Request parameters: +Path parameters: project_key -Body Parameters: -Name Type Required Default Value Description +Body parameters: +Name Type Required Default Value Remarks nodes array Yes Nodes node_key string No Node key tags array No Original node tag list @@ -26,92 +26,90 @@ operations array Yes tags array No Operation tag list mode string No Operation type ADD / DELETE -Return Data: -Name Type Required Default Value Description +Return data: +Name Type Required Default Value Remarks code integer Yes Status code -msg string Yes Message -data object Yes Return data +msg string Yes Prompt message +data object Yes Returned data list array No Node list true / false -node_type string No Node type DATASET / RECIPE +node_type string No Node type DATASET / RECIPE node_key string No Node key ``` -Unit Test: +Unit test: ```python @pytest.mark.parametrize( "project_key, nodes, operations, expected_msg", [ ("project_key", [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "success"), ("project_key", [{"node_key": "dataset_002", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["tag1"], "mode": "DELETE"}], "success"), -("", [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Missing necessary parameter project_key"), +("", [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Missing the required parameter project_key"), (123, [{"node_key": "dataset_001", "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Incorrect parameter type"), ("project_key", [{"node_key": "a"*201, "tags": ["tag1", "tag2"], "node_type": "DATASET"}], [{"tags": ["new_tag1"], "mode": "ADD"}], "Request parameter exceeds field boundary") ] ) def test_node_tags(project_key, nodes, operations, expected_msg): pass -``` -The above is an example of interface definition and unit test. -Next, please act as an expert test manager with 20 years of experience at Google. -After I provide the interface definition, please reply with the unit test. -There are a few requirements: -1. Only output one `@pytest.mark.parametrize` and the corresponding test_ function - (with a pass inside, not implemented). - -- The function parameters should include expected_msg for result validation. -2. The generated test cases should use shorter text or numbers and be as concise as possible. -3. If comments are needed, use Chinese. -If you understand, please wait for me to provide the interface definition -and only reply with "Understood" to save tokens. -''' +# The above is an interface definition and a unit test example. +# Next, please play the role of an expert test manager with 20 years of experience at Google. When I give the interface definition, +# reply to me with a unit test. There are several requirements: +# 1. Only output one `@pytest.mark.parametrize` and the corresponding test_ function (inside pass, do not implement). +# -- The function parameter contains expected_msg for result verification. +# 2. The generated test cases use shorter text or numbers and are as compact as possible. +# 3. If comments are needed, use Chinese. -ACT_PROMPT_PREFIX = '''Reference test types: such as missing request parameters, field boundary checks, incorrect field types. -Please output 10 test cases within a `@pytest.mark.parametrize` scope. +# If you understand, please wait for me to give the interface definition and just answer "Understood" to save tokens. + +ACT_PROMPT_PREFIX = '''Refer to the test types: such as missing request parameters, field boundary verification, incorrect field type. +Please output 10 test cases within one `@pytest.mark.parametrize` scope. ```text ''' -YFT_PROMPT_PREFIX = '''Reference test types: such as SQL injection, cross-site scripting (XSS), illegal access and unauthorized access, authentication and authorization, parameter verification, exception handling, file upload and download. -Please output 10 test cases within a `@pytest.mark.parametrize` scope. +YFT_PROMPT_PREFIX = '''Refer to the test types: such as SQL injection, cross-site scripting (XSS), unauthorized access and privilege escalation, +authentication and authorization, parameter verification, exception handling, file upload and download. +Please output 10 test cases within one `@pytest.mark.parametrize` scope. ```text ''' + OCR_API_DOC = '''```text -API Name: OCR Recognition -API Path: /api/v1/contract/treaty/task/ocr -Method: POST +Interface Name: OCR recognition +Interface Path: /api/v1/contract/treaty/task/ocr +Method: POST Request Parameters: Path Parameters: Body Parameters: -Name Type Mandatory Default Value Remarks +Name Type Required Default Value Remarks file_id string Yes box array Yes -contract_id number Yes Contract ID +contract_id number Yes Contract id start_time string No yyyy-mm-dd end_time string No yyyy-mm-dd -extract_type number No Recognition Type 1- During Import 2- After Import, Default is 1 +extract_type number No Recognition type 1- During import 2- After import Default 1 -Return Data: -Name Type Mandatory Default Value Remarks +Response Data: +Name Type Required Default Value Remarks code integer Yes message string Yes data object Yes - -''' + + class UTGenerator: - """UT Generator: Constructs UT from API documentation.""" + """UT Generator: Construct UT through API documentation""" def __init__(self, swagger_file: str, ut_py_path: str, questions_path: str, chatgpt_method: str = "API", template_prefix=YFT_PROMPT_PREFIX) -> None: - """Initializes the UT generator. + """Initialize UT Generator Args: - swagger_file: Path to the swagger. - ut_py_path: Path to store test cases. - questions_path: Path to store templates for further investigation. - chatgpt_method: API. - template_prefix: Use template, defaults to YFT_UT_PROMPT. + swagger_file: path to the swagger file + ut_py_path: path to store test cases + questions_path: path to store the template, facilitating subsequent checks + chatgpt_method: API method + template_prefix: use the template, default is YFT_UT_PROMPT """ self.swagger_file = swagger_file self.ut_py_path = ut_py_path @@ -119,12 +117,12 @@ class UTGenerator: assert chatgpt_method in ["API"], "Invalid chatgpt_method" self.chatgpt_method = chatgpt_method - # ICL: In-Context Learning. Here, an example is provided for GPT to follow. + # ICL: In-Context Learning, provide an example here for GPT to mimic self.icl_sample = ICL_SAMPLE self.template_prefix = template_prefix def get_swagger_json(self) -> dict: - """Loads Swagger JSON from a local file.""" + """Load Swagger JSON from a local file""" with open(self.swagger_file, "r", encoding="utf-8") as file: swagger_json = json.load(file) return swagger_json @@ -145,30 +143,30 @@ class UTGenerator: return self.__para_to_str(prop, required, name) def build_object_properties(self, node, prop_object_required, level: int = 0) -> str: - """Recursively outputs properties of object and array[object] types. + """Recursively output properties of object and array[object] types Args: - node: Value of the sub-item. - prop_object_required: Whether it's a required item. - level: Current recursion depth. + node (_type_): value of the child item + prop_object_required (_type_): whether it's a required field + level: current recursion depth """ doc = "" def dive_into_object(node): - """If it's an object type, recursively outputs its properties.""" + """If it's an object type, recursively output its properties""" if node.get("type") == "object": sub_properties = node.get("properties", {}) return self.build_object_properties(sub_properties, prop_object_required, level=level + 1) return "" if node.get("in", "") in ["query", "header", "formData"]: - doc += f'{"\t" * level}{self._para_to_str(node)}\n' + doc += f'{" " * level}{self._para_to_str(node)}\n' doc += dive_into_object(node) return doc for name, prop in node.items(): - doc += f'{"\t" * level}{self.para_to_str(name, prop, prop_object_required)}\n' + doc += f'{" " * level}{self.para_to_str(name, prop, prop_object_required)}\n' doc += dive_into_object(prop) if prop["type"] == "array": items = prop.get("items", {}) @@ -176,10 +174,10 @@ class UTGenerator: return doc def get_tags_mapping(self) -> dict: - """Handles tag and path mapping. + """Process tag and path mappings Returns: - Dict: Mapping of tag to path. + Dict: mapping of tag to path """ swagger_data = self.get_swagger_json() paths = swagger_data["paths"] @@ -197,7 +195,7 @@ class UTGenerator: return tags def generate_ut(self, include_tags) -> bool: - """Generates test case files.""" + """Generate test case files""" tags = self.get_tags_mapping() for tag, paths in tags.items(): if include_tags is None or tag in include_tags: @@ -207,19 +205,19 @@ class UTGenerator: def build_api_doc(self, node: dict, path: str, method: str) -> str: summary = node["summary"] - doc = f"Interface name: {summary}\nInterface path: {path}\nMethod: {method.upper()}\n" - doc += "\nRequest parameters:\n" + doc = f"API Name: {summary}\nAPI Path: {path}\nMethod: {method.upper()}\n" + doc += "\nRequest Parameters:\n" if "parameters" in node: parameters = node["parameters"] - doc += "Path parameters:\n" + doc += "Path Parameters:\n" # param["in"]: path / formData / body / query / header for param in parameters: if param["in"] == "path": doc += f'{param["name"]} \n' - doc += "\nBody parameters:\n" - doc += "Name\tType\tRequired\tDefault\tNotes\n" + doc += "\nBody Parameters:\n" + doc += "Name\tType\tRequired\tDefault Value\tRemarks\n" for param in parameters: if param["in"] == "body": schema = param.get("schema", {}) @@ -229,9 +227,9 @@ class UTGenerator: else: doc += self.build_object_properties(param, []) - # Output return data information - doc += "\nReturn data:\n" - doc += "Name\tType\tRequired\tDefault\tNotes\n" + # Display response data information + doc += "\nResponse Data:\n" + doc += "Name\tType\tRequired\tDefault Value\tRemarks\n" responses = node["responses"] response = responses.get("200", {}) schema = response.get("schema", {}) @@ -251,7 +249,7 @@ class UTGenerator: file.write(data) def ask_gpt_and_save(self, question: str, tag: str, fname: str): - """Generate a question and save both the question and answer.""" + """Generate questions and store both questions and answers""" messages = [self.icl_sample, question] result = self.gpt_msgs_to_code(messages=messages) @@ -259,11 +257,11 @@ class UTGenerator: self._store(result, self.ut_py_path, tag, f"{fname}.py") def _generate_ut(self, tag, paths): - """Process the structure under the data path. + """Process the structure under a data path Args: - tag: Module name. - paths: Path Object. + tag (_type_): module name + paths (_type_): Path Object """ for path, path_obj in paths.items(): for method, node in path_obj.items(): @@ -273,7 +271,7 @@ class UTGenerator: self.ask_gpt_and_save(question, tag, summary) def gpt_msgs_to_code(self, messages: list) -> str: - """Choose based on different call methods.""" + """Choose based on different calling methods""" result = '' if self.chatgpt_method == "API": result = GPTAPI().ask_code(msgs=messages) @@ -281,11 +279,11 @@ class UTGenerator: return result def get_file_path(self, base: Path, fname: str): - """Save to different file paths. + """Save different file paths Args: - base (str): Path. - fname (str): File name. + base (str): Path + fname (str): File name """ path = Path(base) path.mkdir(parents=True, exist_ok=True) diff --git a/metagpt/tools/web_browser_engine.py b/metagpt/tools/web_browser_engine.py index d1f83934f..67b794dd1 100644 --- a/metagpt/tools/web_browser_engine.py +++ b/metagpt/tools/web_browser_engine.py @@ -57,3 +57,4 @@ def get_page_content(page: str): if __name__ == "__main__": text = asyncio.run(WebBrowserEngine().run("https://fuzhi.ai/")) print(text) + \ No newline at end of file diff --git a/metagpt/tools/web_browser_engine_playwright.py b/metagpt/tools/web_browser_engine_playwright.py index ae8644cce..94539e9a3 100644 --- a/metagpt/tools/web_browser_engine_playwright.py +++ b/metagpt/tools/web_browser_engine_playwright.py @@ -118,4 +118,4 @@ if __name__ == "__main__": for i in ("chromium", "firefox", "webkit"): text = asyncio.run(PlaywrightWrapper(i).run("https://httpbin.org/ip")) print(text) - print(i) + print(i) \ No newline at end of file diff --git a/metagpt/tools/web_browser_engine_selenium.py b/metagpt/tools/web_browser_engine_selenium.py index bd8a456ea..78533e05a 100644 --- a/metagpt/tools/web_browser_engine_selenium.py +++ b/metagpt/tools/web_browser_engine_selenium.py @@ -109,3 +109,4 @@ def _gen_get_driver_func(browser_type, *args, executable_path=None): if __name__ == "__main__": text = asyncio.run(SeleniumWrapper("chrome").run("https://fuzhi.ai/")) print(text) + \ No newline at end of file diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 399ff22de..b61dbc2eb 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -15,10 +15,9 @@ from metagpt.logs import logger def check_cmd_exists(command) -> int: - """Check if a command exists. - - :param command: Command to check. - :return: Returns 0 if the command exists, otherwise returns a non-zero value. + """ Check if the command exists + :param command: Command to be checked + :return: Returns 0 if the command exists, non-zero otherwise """ check_command = 'command -v ' + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }' result = os.system(check_command) @@ -29,19 +28,19 @@ class OutputParser: @classmethod def parse_blocks(cls, text: str): - # First, split the text into different blocks based on "##". + # First, split the text into different blocks based on "##" blocks = text.split("##") - # Create a dictionary to store the title and content of each block. + # Create a dictionary to store the title and content of each block block_dict = {} - # Iterate through all blocks. + # Iterate through all blocks for block in blocks: - # If the block is not empty, continue processing. + # If the block is not empty, continue processing if block.strip() != "": - # Split the block's title and content and trim whitespace. + # Separate the title and content of the block and trim whitespace block_title, block_content = block.split("\n", 1) - # LLM might make mistakes; correct it here. + # LLM might have an error, correct it here if block_title[-1] == ":": block_title = block_title[:-1] block_dict[block_title.strip()] = block_content.strip() @@ -85,13 +84,13 @@ class OutputParser: block_dict = cls.parse_blocks(data) parsed_data = {} for block, content in block_dict.items(): - # Try to remove the code marker. + # Try to remove the code marker try: content = cls.parse_code(text=content) except Exception: pass - # Try to parse the list. + # Try to parse the list try: content = cls.parse_file_list(text=content) except Exception: @@ -104,7 +103,7 @@ class OutputParser: block_dict = cls.parse_blocks(data) parsed_data = {} for block, content in block_dict.items(): - # Try to remove the code marker. + # Try to remove the code marker try: content = cls.parse_code(text=content) except Exception: @@ -115,11 +114,18 @@ class OutputParser: else: typing = typing_define if typing == List[str] or typing == List[Tuple[str, str]]: - # Try to parse the list. + # Try to parse the list try: content = cls.parse_file_list(text=content) except Exception: pass + # TODO: Removing extra quotes is risky, will address later + # elif typing == str: + # # Try to remove extra quotes + # try: + # content = cls.parse_str(text=content) + # except Exception: + # pass parsed_data[block] = content return parsed_data @@ -136,17 +142,17 @@ class CodeParser: @classmethod def parse_blocks(cls, text: str): - # First, split the text into different blocks based on "##". + # First, split the text into different blocks based on "##" blocks = text.split("##") - # Create a dictionary to store the title and content of each block. + # Create a dictionary to store the title and content of each block block_dict = {} - # Iterate through all blocks. + # Iterate through all blocks for block in blocks: - # If the block is not empty, continue processing. + # If the block is not empty, continue processing if block.strip() != "": - # Split the block's title and content and trim whitespace. + # Separate the title and content of the block and trim whitespace block_title, block_content = block.split("\n", 1) block_dict[block_title.strip()] = block_content.strip() diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index ca3a928ea..0330aca0a 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -8,7 +8,7 @@ import os import subprocess from pathlib import Path - +from metagpt.config import CONFIG from metagpt.const import PROJECT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists @@ -39,11 +39,15 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height # Call the `mmdc` command to convert the Mermaid code to a PNG logger.info(f"Generating {output_file}..") if IS_DOCKER == 'true': - subprocess.run(['mmdc', '-p', '/app/metagpt/puppeteer-config.json', '-i', + subprocess.run(['mmdc', '-p', '/app/metagpt/config/puppeteer-config.json', '-i', str(tmp), '-o', output_file, '-w', str(width), '-H', str(height)]) else: - subprocess.run(['mmdc', '-i', str(tmp), '-o', - output_file, '-w', str(width), '-H', str(height)]) + if CONFIG.puppeteer_config: + subprocess.run([CONFIG.mmdc, '-p', CONFIG.puppeteer_config, '-i', str(tmp), '-o', + output_file, '-w', str(width), '-H', str(height)]) + else: + subprocess.run([CONFIG.mmdc, '-i', str(tmp), '-o', + output_file, '-w', str(width), '-H', str(height)]) return 0 @@ -102,3 +106,4 @@ if __name__ == '__main__': # logger.info(print_members(print_members)) mermaid_to_file(MMC1, PROJECT_ROOT / 'tmp/1.png') mermaid_to_file(MMC2, PROJECT_ROOT / 'tmp/2.png') + \ No newline at end of file diff --git a/metagpt/utils/read_document.py b/metagpt/utils/read_document.py index 0f14b9047..c837baf25 100644 --- a/metagpt/utils/read_document.py +++ b/metagpt/utils/read_document.py @@ -9,13 +9,13 @@ import docx def read_docx(file_path: str) -> list: - """Open and read a docx file.""" + """Open a docx file""" doc = docx.Document(file_path) - # Create an empty list to store paragraph contents. + # Create an empty list to store paragraph contents paragraphs_list = [] - # Iterate through the paragraphs in the document and add their content to the list. + # Iterate through the paragraphs in the document and add their content to the list for paragraph in doc.paragraphs: paragraphs_list.append(paragraph.text) diff --git a/metagpt/utils/singleton.py b/metagpt/utils/singleton.py index a9e0862c0..474b537db 100644 --- a/metagpt/utils/singleton.py +++ b/metagpt/utils/singleton.py @@ -20,3 +20,4 @@ class Singleton(abc.ABCMeta, type): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] + \ No newline at end of file diff --git a/startup.py b/startup.py index e062babb5..c990aa66d 100644 --- a/startup.py +++ b/startup.py @@ -34,3 +34,4 @@ def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool if __name__ == '__main__': fire.Fire(main) + \ No newline at end of file From 5a466d577fa0746616e1e9c73ede520b374d9f7f Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 2 Aug 2023 16:22:35 -0500 Subject: [PATCH 06/27] fixed the conflicts --- metagpt/config.py | 5 +- metagpt/roles/role.py | 138 +++++++++++++++++++++--------------------- 2 files changed, 73 insertions(+), 70 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index e479ab018..3753bb3b0 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -30,7 +30,7 @@ class Config(metaclass=Singleton): """ Regular usage method: config = Config("config.yaml") - secret_key = config.get("MY_SECRET_KEY") + secret_key = config.get_key("MY_SECRET_KEY") print("Secret key:", secret_key) """ @@ -79,6 +79,9 @@ class Config(metaclass=Singleton): self.total_cost = 0.0 self.puppeteer_config = self._get("PUPPETEER_CONFIG","") self.mmdc = self._get("MMDC","mmdc") + self.update_costs = self._get("UPDATE_COSTS",True) + self.calc_usage = self._get("CALC_USAGE",True) + def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index bfdd7f18a..fa2971ae7 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -77,7 +77,7 @@ class RoleContext(BaseModel): def check(self, role_id: str): if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: self.long_term_memory.recover_memory(role_id, self) - self.memory = self.long_term_memory # use memory to act as long_term_memory for unified operation + self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation @property def important_memory(self) -> list[Message]: @@ -158,85 +158,85 @@ class Role: next_state = "0" self._set_state(int(next_state)) -async def _act(self) -> Message: - # prompt = self.get_prefix() - # prompt += ROLE_TEMPLATE.format(name=self.profile, state=self.states[self.state], result=response, - # history=self.history) + async def _act(self) -> Message: + # prompt = self.get_prefix() + # prompt += ROLE_TEMPLATE.format(name=self.profile, state=self.states[self.state], result=response, + # history=self.history) - logger.info(f"{self._setting}: ready to {self._rc.todo}") - response = await self._rc.todo.run(self._rc.important_memory) - # logger.info(response) - if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) - else: - msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) - self._rc.memory.add(msg) - # logger.debug(f"{response}") + logger.info(f"{self._setting}: ready to {self._rc.todo}") + response = await self._rc.todo.run(self._rc.important_memory) + # logger.info(response) + if isinstance(response, ActionOutput): + msg = Message(content=response.content, instruct_content=response.instruct_content, + role=self.profile, cause_by=type(self._rc.todo)) + else: + msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) + self._rc.memory.add(msg) + # logger.debug(f"{response}") - return msg + return msg -async def _observe(self) -> int: - """Observe from the environment, obtain important information, and add it to memory""" - if not self._rc.env: - return 0 - env_msgs = self._rc.env.memory.get() + async def _observe(self) -> int: + """Observe from the environment, obtain important information, and add it to memory""" + if not self._rc.env: + return 0 + env_msgs = self._rc.env.memory.get() - observed = self._rc.env.memory.get_by_actions(self._rc.watch) + observed = self._rc.env.memory.get_by_actions(self._rc.watch) - news = self._rc.memory.remember(observed) # remember recent exact or similar memories + news = self._rc.memory.remember(observed) # remember recent exact or similar memories - for i in env_msgs: - self.recv(i) + for i in env_msgs: + self.recv(i) - news_text = [f"{i.role}: {i.content[:20]}..." for i in news] - if news_text: - logger.debug(f'{self._setting} observed: {news_text}') - return len(news) + news_text = [f"{i.role}: {i.content[:20]}..." for i in news] + if news_text: + logger.debug(f'{self._setting} observed: {news_text}') + return len(news) -def _publish_message(self, msg): - """If the role belongs to env, then the role's messages will be broadcast to env""" - if not self._rc.env: - # If env does not exist, do not publish the message - return - self._rc.env.publish_message(msg) + def _publish_message(self, msg): + """If the role belongs to env, then the role's messages will be broadcast to env""" + if not self._rc.env: + # If env does not exist, do not publish the message + return + self._rc.env.publish_message(msg) -async def _react(self) -> Message: - """Think first, then act""" - await self._think() - logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") - return await self._act() + async def _react(self) -> Message: + """Think first, then act""" + await self._think() + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + return await self._act() -def recv(self, message: Message) -> None: - """add message to history.""" - # self._history += f"\n{message}" - # self._context = self._history - if message in self._rc.memory.get(): - return - self._rc.memory.add(message) + def recv(self, message: Message) -> None: + """add message to history.""" + # self._history += f"\n{message}" + # self._context = self._history + if message in self._rc.memory.get(): + return + self._rc.memory.add(message) -async def handle(self, message: Message) -> Message: - """Receive information and reply with actions""" - # logger.debug(f"{self.name=}, {self.profile=}, {message.role=}") - self.recv(message) + async def handle(self, message: Message) -> Message: + """Receive information and reply with actions""" + # logger.debug(f"{self.name=}, {self.profile=}, {message.role=}") + self.recv(message) - return await self._react() + return await self._react() -async def run(self, message=None): - """Observe, and think and act based on the results of the observation""" - if message: - if isinstance(message, str): - message = Message(message) - if isinstance(message, Message): - self.recv(message) - if isinstance(message, list): - self.recv(Message("\n".join(message))) - elif not await self._observe(): - # If there is no new information, suspend and wait - logger.debug(f"{self._setting}: no news. waiting.") - return + async def run(self, message=None): + """Observe, and think and act based on the results of the observation""" + if message: + if isinstance(message, str): + message = Message(message) + if isinstance(message, Message): + self.recv(message) + if isinstance(message, list): + self.recv(Message("\n".join(message))) + elif not await self._observe(): + # If there is no new information, suspend and wait + logger.debug(f"{self._setting}: no news. waiting.") + return - rsp = await self._react() - # Publish the reply to the environment, waiting for the next subscriber to process - self._publish_message(rsp) - return rsp + rsp = await self._react() + # Publish the reply to the environment, waiting for the next subscriber to process + self._publish_message(rsp) + return rsp From 5170ee720a666d2f6cab4d98a45754e69ce65ded Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 2 Aug 2023 17:09:05 -0500 Subject: [PATCH 07/27] roles update. More comments and docstrings --- metagpt/roles/architect.py | 24 ++++++- metagpt/roles/engineer.py | 110 ++++++++++++++++++------------- metagpt/roles/product_manager.py | 29 ++++++-- metagpt/roles/project_manager.py | 29 ++++++-- metagpt/roles/qa_engineer.py | 25 ++++++- metagpt/roles/seacher.py | 39 +++++++++-- 6 files changed, 193 insertions(+), 63 deletions(-) diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 5fc7bdef7..d0756672e 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -11,10 +11,28 @@ from metagpt.roles import Role class Architect(Role): - """Architect: Listen to PRD, responsible for designing API, designing code files""" - def __init__(self, name="Bob", profile="Architect", goal="Design a concise, usable, complete python system", - constraints="Try to specify good open source tools as much as possible"): + """ + Represents an Architect role in a software development process. + + Attributes: + name (str): Name of the architect. + profile (str): Role profile, default is 'Architect'. + goal (str): Primary goal or responsibility of the architect. + constraints (str): Constraints or guidelines for the architect. + """ + + def __init__(self, + name: str = "Bob", + profile: str = "Architect", + goal: str = "Design a concise, usable, complete python system", + constraints: str = "Try to specify good open source tools as much as possible") -> None: + """Initializes the Architect with given attributes.""" super().__init__(name, profile, goal, constraints) + + # Initialize actions specific to the Architect role self._init_actions([WriteDesign]) + + # Set events or actions the Architect should watch or be aware of self._watch({WritePRD}) + \ No newline at end of file diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 7567b2ed9..a9963c25b 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -46,9 +46,27 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): - def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", - constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", - n_borg=1, use_code_review=False): + """ + Represents an Engineer role responsible for writing and possibly reviewing code. + + Attributes: + name (str): Name of the engineer. + profile (str): Role profile, default is 'Engineer'. + goal (str): Goal of the engineer. + constraints (str): Constraints for the engineer. + n_borg (int): Number of borgs. + use_code_review (bool): Whether to use code review. + todos (list): List of tasks. + """ + + def __init__(self, + name: str = "Alex", + profile: str = "Engineer", + goal: str = "Write elegant, readable, extensible, efficient code", + constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", + n_borg: int = 1, + use_code_review: bool = False) -> None: + """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review @@ -142,47 +160,47 @@ class Engineer(Role): msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) return msg -async def _act_sp_precision(self) -> Message: - for todo in self.todos: - """ - # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): - 1. All from Architect - 2. All from ProjectManager - 3. Do we need other codes (currently needed)? - TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. - """ - context = [] - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - for m in msg: - context.append(m.content) - context_str = "\n".join(context) - # Write code - code = await WriteCode().run( - context=context_str, - filename=todo - ) - # Code review + async def _act_sp_precision(self) -> Message: + for todo in self.todos: + """ + # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): + 1. All from Architect + 2. All from ProjectManager + 3. Do we need other codes (currently needed)? + TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. + """ + context = [] + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + for m in msg: + context.append(m.content) + context_str = "\n".join(context) + # Write code + code = await WriteCode().run( + context=context_str, + filename=todo + ) + # Code review + if self.use_code_review: + try: + rewrite_code = await WriteCodeReview().run( + context=context_str, + code=code, + filename=todo + ) + code = rewrite_code + except Exception as e: + logger.error("code review failed!", e) + pass + self.write_file(todo, code) + msg = Message(content=code, role=self.profile, cause_by=WriteCode) + self._rc.memory.add(msg) + + logger.info(f'Done {self.get_workspace()} generating.') + msg = Message(content="all done.", role=self.profile, cause_by=WriteCode) + return msg + + async def _act(self) -> Message: + """Determines the mode of action based on whether code review is used.""" if self.use_code_review: - try: - rewrite_code = await WriteCodeReview().run( - context=context_str, - code=code, - filename=todo - ) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) - pass - self.write_file(todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) - self._rc.memory.add(msg) - - logger.info(f'Done {self.get_workspace()} generating.') - msg = Message(content="all done.", role=self.profile, cause_by=WriteCode) - return msg - -async def _act(self) -> Message: - if self.use_code_review: - return await self._act_sp_precision() - return await self._act_sp() - \ No newline at end of file + return await self._act_sp_precision() + return await self._act_sp() \ No newline at end of file diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index b89aac28c..9996e907a 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -10,9 +10,30 @@ from metagpt.roles import Role class ProductManager(Role): - def __init__(self, name="Alice", profile="Product Manager", goal="Efficiently create a successful product", - constraints=""): + """ + Represents a Product Manager role responsible for product development and management. + + Attributes: + name (str): Name of the product manager. + profile (str): Role profile, default is 'Product Manager'. + goal (str): Goal of the product manager. + constraints (str): Constraints or limitations for the product manager. + """ + + def __init__(self, + name: str = "Alice", + profile: str = "Product Manager", + goal: str = "Efficiently create a successful product", + constraints: str = "") -> None: + """ + Initializes the ProductManager role with given attributes. + + Args: + name (str): Name of the product manager. + profile (str): Role profile. + goal (str): Goal of the product manager. + constraints (str): Constraints or limitations for the product manager. + """ super().__init__(name, profile, goal, constraints) self._init_actions([WritePRD]) - self._watch([BossRequirement]) - \ No newline at end of file + self._watch([BossRequirement]) \ No newline at end of file diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 0ad871b4c..dd4ba42ae 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -10,9 +10,30 @@ from metagpt.roles import Role class ProjectManager(Role): - def __init__(self, name="Eve", profile="Project Manager", - goal="Improve team efficiency and deliver with quality and quantity", constraints=""): + """ + Represents a Project Manager role responsible for overseeing project execution and team efficiency. + + Attributes: + name (str): Name of the project manager. + profile (str): Role profile, default is 'Project Manager'. + goal (str): Goal of the project manager. + constraints (str): Constraints or limitations for the project manager. + """ + + def __init__(self, + name: str = "Eve", + profile: str = "Project Manager", + goal: str = "Improve team efficiency and deliver with quality and quantity", + constraints: str = "") -> None: + """ + Initializes the ProjectManager role with given attributes. + + Args: + name (str): Name of the project manager. + profile (str): Role profile. + goal (str): Goal of the project manager. + constraints (str): Constraints or limitations for the project manager. + """ super().__init__(name, profile, goal, constraints) self._init_actions([WriteTasks]) - self._watch([WriteDesign]) - \ No newline at end of file + self._watch([WriteDesign]) \ No newline at end of file diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 70968761f..1dfb0dc6e 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -10,6 +10,29 @@ from metagpt.roles import Role class QaEngineer(Role): - def __init__(self, name, profile, goal, constraints): + """ + Represents a Quality Assurance (QA) Engineer role responsible for writing tests to ensure software quality. + + Attributes: + name (str): Name of the QA engineer. + profile (str): Role profile. + goal (str): Goal of the QA engineer. + constraints (str): Constraints or limitations for the QA engineer. + """ + + def __init__(self, + name: str, + profile: str, + goal: str, + constraints: str) -> None: + """ + Initializes the QaEngineer role with given attributes. + + Args: + name (str): Name of the QA engineer. + profile (str): Role profile. + goal (str): Goal of the QA engineer. + constraints (str): Constraints or limitations for the QA engineer. + """ super().__init__(name, profile, goal, constraints) self._init_actions([WriteTest]) \ No newline at end of file diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index da617974e..0b6e089da 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -13,26 +13,55 @@ from metagpt.tools import SearchEngineType class Searcher(Role): - def __init__(self, name='Alice', profile='Smart Assistant', goal='Provide search services for users', - constraints='Answer is rich and complete', engine=SearchEngineType.SERPAPI_GOOGLE, **kwargs): + """ + Represents a Searcher role responsible for providing search services to users. + + Attributes: + name (str): Name of the searcher. + profile (str): Role profile. + goal (str): Goal of the searcher. + constraints (str): Constraints or limitations for the searcher. + engine (SearchEngineType): The type of search engine to use. + """ + + def __init__(self, + name: str = 'Alice', + profile: str = 'Smart Assistant', + goal: str = 'Provide search services for users', + constraints: str = 'Answer is rich and complete', + engine=SearchEngineType.SERPAPI_GOOGLE, + **kwargs) -> None: + """ + Initializes the Searcher role with given attributes. + + Args: + name (str): Name of the searcher. + profile (str): Role profile. + goal (str): Goal of the searcher. + constraints (str): Constraints or limitations for the searcher. + engine (SearchEngineType): The type of search engine to use. + """ super().__init__(name, profile, goal, constraints, **kwargs) self._init_actions([SearchAndSummarize(engine=engine)]) def set_search_func(self, search_func): + """Sets a custom search function for the searcher.""" action = SearchAndSummarize("", engine=SearchEngineType.CUSTOM_ENGINE, search_func=search_func) self._init_actions([action]) async def _act_sp(self) -> Message: + """Performs the search action in a single process.""" logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.memory.get(k=0)) - # logger.info(response) + if isinstance(response, ActionOutput): msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + role=self.profile, cause_by=type(self._rc.todo)) else: msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) + return msg async def _act(self) -> Message: + """Determines the mode of action for the searcher.""" return await self._act_sp() - \ No newline at end of file From b8d6a1ad4597283877ce7e308d2aac9b87944b65 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:30:48 -0500 Subject: [PATCH 08/27] Update debug_error.py --- metagpt/actions/debug_error.py | 46 +++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 6a7cbc872..c955f30ea 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -5,16 +5,56 @@ @Author : alexanderwu @File : debug_error.py """ +import re + +from metagpt.logs import logger from metagpt.actions.action import Action +from metagpt.utils.common import CodeParser - +PROMPT_TEMPLATE = """ +NOTICE +1. Role: You are a Development Engineer or QA engineer; +2. Task: You received this message from another Development Engineer or QA engineer who ran or tested your code. +Based on the message, first, figure out your own role, i.e. Engineer or QaEngineer, +then rewrite the development code or the test code based on your role, the error, and the summary, such that all bugs are fixed and the code performs well. +Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. +The message is as follows: +{context} +--- +Now you should start rewriting the code: +## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. +""" class DebugError(Action): - def __init__(self, name, context=None, llm=None): + def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) +<<<<<<< main async def run(self, code, error): prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ f"\n\n{error}\n\nPlease try to fix the error in this code." fixed_code = await self._aask(prompt) return fixed_code - \ No newline at end of file + +======= + # async def run(self, code, error): + # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ + # f"\n\n{error}\n\nPlease try to fix the error in this code." + # fixed_code = await self._aask(prompt) + # return fixed_code + + async def run(self, context): + if "PASS" in context: + return "", "the original code works fine, no need to debug" + + file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1) + + logger.info(f"Debug and rewrite {file_name}") + + prompt = PROMPT_TEMPLATE.format(context=context) + + rsp = await self._aask(prompt) + + code = CodeParser.parse_code(block="", text=rsp) + + return file_name, code +>>>>>>> main From 14edcbc041b8641ddeef6022e5cc4fa68b2e1185 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:31:08 -0500 Subject: [PATCH 09/27] Update run_code.py --- metagpt/actions/run_code.py | 62 ++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 5b3106f4d..7fcf20975 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -6,21 +6,75 @@ @File : run_code.py """ import traceback +import os +import subprocess +from typing import List, Tuple +from metagpt.logs import logger from metagpt.actions.action import Action +PROMPT_TEMPLATE = """ +Role: You are a senior development and qa engineer, your role is summarize the code running result. +If the running result does not include an error, you should explicitly approve the result. +On the other hand, if the running result indicates some error, you should point out which part, the development code or the test code, produces the error, +and give specific instructions on fixing the errors. Here is the code info: +{context} +Now you should begin your analysis +--- +## instruction: +Please summarize the cause of the errors and give correction instruction +## File To Rewrite: +Determine the ONE file to rewrite in order to fix the error, for example, xyz.py, or test_xyz.py +## Status: +Determine if all of the code works fine, if so write PASS, else FAIL, +WRITE ONLY ONE WORD, PASS OR FAIL, IN THI SECTION +## Send To: +Please write Engineer if the errors are due to problematic development codes, and QaEngineer to problematic test codes, and NoOne if there are no errors, +WRITE ONLY ONE WORD, Engineer OR QaEngineer OR NoOne, IN THIS SECTION. +--- +You should fill in necessary instruction, status, send to, and finally return all content between the --- segment line. +""" + +CONTEXT = """ +## Development Code File Name +{code_file_name} +## Development Code +```python +{code} +``` +## Test File Name +{test_file_name} +## Test Code +```python +{test_code} +``` +## Running Command +{command} +## Running Output +standard output: {outs}; +standard errors: {errs}; +""" class RunCode(Action): - def __init__(self, name, context=None, llm=None): + def __init__(self, name="RunCode", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, code): + @classmethod + async def run_text(cls, code) -> Tuple[str, str]: try: # We will document_store the result in this dictionary namespace = {} exec(code, namespace) - return namespace.get('result', None) + return namespace.get('result', ""), "" except Exception: # If there is an error in the code, return the error message +<<<<<<< main return traceback.format_exc() - \ No newline at end of file + +======= + return "", traceback.format_exc() + + @classmethod + async def run_script(cls, working_directory, additional_python_paths=[], command=[]) -> Tuple[str, str]: + working_directory = str(working_directory) + additional_python_paths = [str(path) for path in additional_python_paths] From 20948cf08e21e960934cd26076300af6374305ca Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:31:25 -0500 Subject: [PATCH 10/27] Update write_test.py --- metagpt/actions/write_test.py | 47 +++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 393b5fcd9..73d4c119d 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -5,23 +5,54 @@ @Author : alexanderwu @File : write_test.py """ +from metagpt.logs import logger from metagpt.actions.action import Action +from metagpt.utils.common import CodeParser +PROMPT_TEMPLATE = """ +NOTICE +1. Role: You are a QA engineer; the main goal is to design, develop, and execute PEP8 compliant, well-structured, maintainable test cases and scripts for Python 3.9. Your focus should be on ensuring the product quality of the entire project through systematic testing. +2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases. +3. Attention1: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script. +4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. +6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail? +7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE. +Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. +----- +## Given the following code, please write appropriate test cases using Python's unittest framework to verify the correctness and robustness of this code: +```python +{code_to_test} +``` +Note that the code to test is at {source_file_path}, we will put your test code at {workspace}/tests/{test_file_name}, and run your test code from {workspace}, +you should correctly import the necessary classes based on these file locations! +## {test_file_name}: Write test code with triple quoto. Do your best to implement THIS ONLY ONE FILE. +""" class WriteTest(Action): - def __init__(self, name="", context=None, llm=None): + def __init__(self, name="WriteTest", context=None, llm=None): super().__init__(name, context, llm) - self.code = None - self.test_prompt_template = """ - Given the following code or function: - {code} - As a test engineer, please write appropriate test cases using Python's unittest framework to verify the correctness and robustness of this code. - """ + async def write_code(self, prompt): + code_rsp = await self._aask(prompt) + code = CodeParser.parse_code(block="", text=code_rsp) + return code +<<<<<<< main async def run(self, code): self.code = code prompt = self.test_prompt_template.format(code=self.code) test_cases = await self._aask(prompt) return test_cases - \ No newline at end of file + +======= + async def run(self, code_to_test, test_file_name, source_file_path, workspace): + prompt = PROMPT_TEMPLATE.format( + code_to_test=code_to_test, + test_file_name=test_file_name, + source_file_path=source_file_path, + workspace=workspace + ) + code = await self.write_code(prompt) + return code +>>>>>>> main From b192d25eecbdcaf21b7e61f364eb9d9f8fd09dd5 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Mon, 7 Aug 2023 09:31:41 -0500 Subject: [PATCH 11/27] Update qa_engineer.py --- metagpt/roles/qa_engineer.py | 163 ++++++++++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 1dfb0dc6e..fe7f037b4 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -5,11 +5,22 @@ @Author : alexanderwu @File : qa_engineer.py """ -from metagpt.actions import WriteTest -from metagpt.roles import Role +import os +import re +from pathlib import Path +from typing import Type +from metagpt.actions import WriteTest, WriteCode, WriteDesign, RunCode, DebugError +from metagpt.const import WORKSPACE_ROOT +from metagpt.logs import logger +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.roles.engineer import Engineer +from metagpt.utils.common import CodeParser, parse_recipient +from metagpt.utils.special_tokens import MSG_SEP, FILENAME_CODE_SEP class QaEngineer(Role): +<<<<<<< main """ Represents a Quality Assurance (QA) Engineer role responsible for writing tests to ensure software quality. @@ -35,4 +46,150 @@ class QaEngineer(Role): constraints (str): Constraints or limitations for the QA engineer. """ super().__init__(name, profile, goal, constraints) - self._init_actions([WriteTest]) \ No newline at end of file + self._init_actions([WriteTest]) +======= + def __init__(self, name="Edward", profile="QaEngineer", + goal="Write comprehensive and robust tests to ensure codes will work as expected without bugs", + constraints="The test code you write should conform to code standard like PEP8, be modular, easy to read and maintain", + test_round_allowed=5): + super().__init__(name, profile, goal, constraints) + self._init_actions([WriteTest]) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates + self._watch([WriteCode, WriteTest, RunCode, DebugError]) + self.test_round = 0 + self.test_round_allowed = test_round_allowed + + @classmethod + def parse_workspace(cls, system_design_msg: Message) -> str: + if not system_design_msg.instruct_content: + return system_design_msg.instruct_content.dict().get("Python package name") + return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) + + def get_workspace(self, return_proj_dir=True) -> Path: + msg = self._rc.memory.get_by_action(WriteDesign)[-1] + if not msg: + return WORKSPACE_ROOT / 'src' + workspace = self.parse_workspace(msg) + # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. + if return_proj_dir: + return WORKSPACE_ROOT / workspace + # development codes directory: workspace/{package_name}/{package_name} + return WORKSPACE_ROOT / workspace / workspace + + def write_file(self, filename: str, code: str): + workspace = self.get_workspace() / 'tests' + file = workspace / filename + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(code) + + async def _write_test(self, message: Message) -> None: + + code_msgs = message.content.split(MSG_SEP) + result_msg_all = [] + for code_msg in code_msgs: + + # write tests + file_name, file_path = code_msg.split(FILENAME_CODE_SEP) + code_to_test = open(file_path, "r").read() + if "test" in file_name: + continue # Engineer might write some test files, skip testing a test file + test_file_name = "test_" + file_name + test_file_path = self.get_workspace() / "tests" / test_file_name + logger.info(f'Writing {test_file_name}..') + test_code = await WriteTest().run( + code_to_test=code_to_test, + test_file_name=test_file_name, + # source_file_name=file_name, + source_file_path=file_path, + workspace=self.get_workspace() + ) + self.write_file(test_file_name, test_code) + + # prepare context for run tests in next round + command = ['python', f'tests/{test_file_name}'] + file_info = { + "file_name": file_name, "file_path": str(file_path), + "test_file_name": test_file_name, "test_file_path": str(test_file_path), + "command": command + } + msg = Message( + content=str(file_info), role=self.profile, cause_by=WriteTest, + sent_from=self.profile, send_to=self.profile + ) + self._publish_message(msg) + + logger.info(f'Done {self.get_workspace()}/tests generating.') + + async def _run_code(self, msg): + file_info = eval(msg.content) + development_file_path = file_info["file_path"] + test_file_path = file_info["test_file_path"] + if not os.path.exists(development_file_path) or not os.path.exists(test_file_path): + return + + development_code = open(development_file_path, "r").read() + test_code = open(test_file_path, "r").read() + proj_dir = self.get_workspace() + development_code_dir = self.get_workspace(return_proj_dir=False) + + result_msg = await RunCode().run( + mode="script", + code=development_code, + code_file_name=file_info["file_name"], + test_code=test_code, + test_file_name=file_info["test_file_name"], + command=file_info["command"], + working_directory=proj_dir, # workspace/package_name, will run tests/test_xxx.py here + additional_python_paths=[development_code_dir], # workspace/package_name/package_name, + # import statement inside package code needs this + ) + + recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself + content = str(file_info) + FILENAME_CODE_SEP + result_msg + msg = Message( + content=content, role=self.profile, cause_by=RunCode, + sent_from=self.profile, send_to=recipient + ) + self._publish_message(msg) + + async def _debug_error(self, msg): + file_info, context = msg.content.split(FILENAME_CODE_SEP) + file_name, code = await DebugError().run(context) + if file_name: + self.write_file(file_name, code) + recipient = msg.sent_from # send back to the one who ran the code for another run, might be one's self + msg = Message(content=file_info, role=self.profile, cause_by=DebugError, sent_from=self.profile, send_to=recipient) + self._publish_message(msg) + + async def _observe(self) -> int: + await super()._observe() + self._rc.news = [msg for msg in self._rc.news \ + if msg.send_to == self.profile] # only relevant msgs count as observed news + return len(self._rc.news) + + async def _act(self) -> Message: + if self.test_round > self.test_round_allowed: + result_msg = Message( + content=f"Exceeding {self.test_round_allowed} rounds of tests, skip (writing code counts as a round, too)", + role=self.profile, cause_by=WriteTest, sent_from=self.profile, send_to="" + ) + return result_msg + + for msg in self._rc.news: + # Decide what to do based on observed msg type, currently defined by human, + # might potentially be moved to _think, that is, let the agent decides for itself + if msg.cause_by == WriteCode: + # engineer wrote a code, time to write a test for it + await self._write_test(msg) + elif msg.cause_by in [WriteTest, DebugError]: + # I wrote or debugged my test code, time to run it + await self._run_code(msg) + elif msg.cause_by == RunCode: + # I ran my test code, time to fix bugs, if any + await self._debug_error(msg) + self.test_round += 1 + result_msg = Message( + content=f"Round {self.test_round} of tests done", + role=self.profile, cause_by=WriteTest, sent_from=self.profile, send_to="" + ) + return result_msg +>>>>>>> main From e3fcdb3177ba37f165f405013128215acdf5d1cd Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:06:11 -0500 Subject: [PATCH 12/27] Update debug_error.py --- metagpt/actions/debug_error.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index c955f30ea..6b9fc744b 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -27,21 +27,7 @@ Now you should start rewriting the code: class DebugError(Action): def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) - -<<<<<<< main - async def run(self, code, error): - prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ - f"\n\n{error}\n\nPlease try to fix the error in this code." - fixed_code = await self._aask(prompt) - return fixed_code - -======= - # async def run(self, code, error): - # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ - # f"\n\n{error}\n\nPlease try to fix the error in this code." - # fixed_code = await self._aask(prompt) - # return fixed_code - + async def run(self, context): if "PASS" in context: return "", "the original code works fine, no need to debug" @@ -57,4 +43,4 @@ class DebugError(Action): code = CodeParser.parse_code(block="", text=rsp) return file_name, code ->>>>>>> main + From 4c85faec112d631b79fed6aab2616f4af9ed732b Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:06:49 -0500 Subject: [PATCH 13/27] Update run_code.py --- metagpt/actions/run_code.py | 50 ++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 7fcf20975..74590968c 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -68,13 +68,55 @@ class RunCode(Action): return namespace.get('result', ""), "" except Exception: # If there is an error in the code, return the error message -<<<<<<< main - return traceback.format_exc() - -======= return "", traceback.format_exc() @classmethod async def run_script(cls, working_directory, additional_python_paths=[], command=[]) -> Tuple[str, str]: working_directory = str(working_directory) additional_python_paths = [str(path) for path in additional_python_paths] + # Copy the current environment variables + env = os.environ.copy() + + # Modify the PYTHONPATH environment variable + additional_python_paths = [working_directory] + additional_python_paths + additional_python_paths = ":".join(additional_python_paths) + env['PYTHONPATH'] = additional_python_paths + ':' + env.get('PYTHONPATH', '') + + # Start the subprocess + process = subprocess.Popen(command, cwd=working_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + + try: + # Wait for the process to complete, with a timeout + stdout, stderr = process.communicate(timeout=10) + except subprocess.TimeoutExpired: + logger.info("The command did not complete within the given timeout.") + process.kill() # Kill the process if it times out + stdout, stderr = process.communicate() + return stdout.decode('utf-8'), stderr.decode('utf-8') + + async def run( + self, code, mode="script", code_file_name="", test_code="", test_file_name="", command=[], **kwargs + ) -> str: + logger.info(f"Running {' '.join(command)}") + if mode == "script": + outs, errs = await self.run_script(command=command, **kwargs) + elif mode == "text": + outs, errs = await self.run_text(code=code) + + logger.info(f"{outs=}") + logger.info(f"{errs=}") + + context = CONTEXT.format( + code=code, code_file_name=code_file_name, + test_code=test_code, test_file_name=test_file_name, + command=" ".join(command), + outs=outs[:500], # outs might be long but they are not important, truncate them to avoid token overflow + errs=errs[:10000] # truncate errors to avoid token overflow + ) + + prompt = PROMPT_TEMPLATE.format(context=context) + rsp = await self._aask(prompt) + + result = context + rsp + + return result From 4b7b418d8d0136ad50fcda48a04137c07dd7e6f1 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:07:25 -0500 Subject: [PATCH 14/27] Update run_code.py --- metagpt/actions/run_code.py | 150 +++++++++++------------------------- 1 file changed, 46 insertions(+), 104 deletions(-) diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 74590968c..60fc92154 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -1,122 +1,64 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/11 17:46 +@Time : 2023/5/11 22:12 @Author : alexanderwu -@File : run_code.py +@File : environment.py """ -import traceback -import os -import subprocess -from typing import List, Tuple +import asyncio +from typing import Iterable -from metagpt.logs import logger -from metagpt.actions.action import Action +from pydantic import BaseModel, Field -PROMPT_TEMPLATE = """ -Role: You are a senior development and qa engineer, your role is summarize the code running result. -If the running result does not include an error, you should explicitly approve the result. -On the other hand, if the running result indicates some error, you should point out which part, the development code or the test code, produces the error, -and give specific instructions on fixing the errors. Here is the code info: -{context} -Now you should begin your analysis ---- -## instruction: -Please summarize the cause of the errors and give correction instruction -## File To Rewrite: -Determine the ONE file to rewrite in order to fix the error, for example, xyz.py, or test_xyz.py -## Status: -Determine if all of the code works fine, if so write PASS, else FAIL, -WRITE ONLY ONE WORD, PASS OR FAIL, IN THI SECTION -## Send To: -Please write Engineer if the errors are due to problematic development codes, and QaEngineer to problematic test codes, and NoOne if there are no errors, -WRITE ONLY ONE WORD, Engineer OR QaEngineer OR NoOne, IN THIS SECTION. ---- -You should fill in necessary instruction, status, send to, and finally return all content between the --- segment line. -""" +from metagpt.memory import Memory +from metagpt.roles import Role +from metagpt.schema import Message -CONTEXT = """ -## Development Code File Name -{code_file_name} -## Development Code -```python -{code} -``` -## Test File Name -{test_file_name} -## Test Code -```python -{test_code} -``` -## Running Command -{command} -## Running Output -standard output: {outs}; -standard errors: {errs}; -""" -class RunCode(Action): - def __init__(self, name="RunCode", context=None, llm=None): - super().__init__(name, context, llm) +class Environment(BaseModel): + """Environment that carries a set of roles. Roles can publish messages to the environment, which can be observed by other roles.""" - @classmethod - async def run_text(cls, code) -> Tuple[str, str]: - try: - # We will document_store the result in this dictionary - namespace = {} - exec(code, namespace) - return namespace.get('result', ""), "" - except Exception: - # If there is an error in the code, return the error message - return "", traceback.format_exc() + roles: dict[str, Role] = Field(default_factory=dict) + memory: Memory = Field(default_factory=Memory) + history: str = Field(default='') - @classmethod - async def run_script(cls, working_directory, additional_python_paths=[], command=[]) -> Tuple[str, str]: - working_directory = str(working_directory) - additional_python_paths = [str(path) for path in additional_python_paths] - # Copy the current environment variables - env = os.environ.copy() + class Config: + arbitrary_types_allowed = True - # Modify the PYTHONPATH environment variable - additional_python_paths = [working_directory] + additional_python_paths - additional_python_paths = ":".join(additional_python_paths) - env['PYTHONPATH'] = additional_python_paths + ':' + env.get('PYTHONPATH', '') + def add_role(self, role: Role): + """Add a Role to the current environment.""" + role.set_env(self) + self.roles[role.profile] = role - # Start the subprocess - process = subprocess.Popen(command, cwd=working_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + def add_roles(self, roles: Iterable[Role]): + """Add a batch of Roles to the current environment.""" + for role in roles: + self.add_role(role) - try: - # Wait for the process to complete, with a timeout - stdout, stderr = process.communicate(timeout=10) - except subprocess.TimeoutExpired: - logger.info("The command did not complete within the given timeout.") - process.kill() # Kill the process if it times out - stdout, stderr = process.communicate() - return stdout.decode('utf-8'), stderr.decode('utf-8') + def publish_message(self, message: Message): + """Publish a message to the current environment.""" + # self.message_queue.put(message) + self.memory.add(message) + self.history += f"\n{message}" - async def run( - self, code, mode="script", code_file_name="", test_code="", test_file_name="", command=[], **kwargs - ) -> str: - logger.info(f"Running {' '.join(command)}") - if mode == "script": - outs, errs = await self.run_script(command=command, **kwargs) - elif mode == "text": - outs, errs = await self.run_text(code=code) + async def run(self, k=1): + """Process the run of all Roles once.""" + # while not self.message_queue.empty(): + # message = self.message_queue.get() + # rsp = await self.manager.handle(message, self) + # self.message_queue.put(rsp) + for _ in range(k): + futures = [] + for role in self.roles.values(): + future = role.run() + futures.append(future) - logger.info(f"{outs=}") - logger.info(f"{errs=}") + await asyncio.gather(*futures) - context = CONTEXT.format( - code=code, code_file_name=code_file_name, - test_code=test_code, test_file_name=test_file_name, - command=" ".join(command), - outs=outs[:500], # outs might be long but they are not important, truncate them to avoid token overflow - errs=errs[:10000] # truncate errors to avoid token overflow - ) + def get_roles(self) -> dict[str, Role]: + """Get all Roles within the environment.""" + return self.roles - prompt = PROMPT_TEMPLATE.format(context=context) - rsp = await self._aask(prompt) - - result = context + rsp - - return result + def get_role(self, name: str) -> Role: + """Get a specified Role within the environment.""" + return self.roles.get(name, None) From 552c22d5da1973c3be0f4f3a9c273df84fe7987f Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:10:08 -0500 Subject: [PATCH 15/27] Update llm.py --- metagpt/llm.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index b8aefec61..c6755919d 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -1,17 +1,22 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/11 14:45 +@Time : 2023/6/1 12:41 @Author : alexanderwu -@File : llm.py +@File : logs.py """ -from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM +import sys -DEFAULT_LLM = LLM() -CLAUDE_LLM = Claude() +from loguru import logger as _logger -async def ai_func(prompt): - """Use LLM for Q&A.""" - return await DEFAULT_LLM.aask(prompt) +from metagpt.const import PROJECT_ROOT + +def define_log_level(print_level="INFO", logfile_level="DEBUG"): + """Adjust log level to above the specified level.""" + _logger.remove() + _logger.add(sys.stderr, level=print_level) + _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) + return _logger + +logger = define_log_level() From 57ffca3b5c4559ac28b257ad5cb45f584769440d Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:10:45 -0500 Subject: [PATCH 16/27] Update llm.py pasted wrong code --- metagpt/llm.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index c6755919d..b8aefec61 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -1,22 +1,17 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/6/1 12:41 +@Time : 2023/5/11 14:45 @Author : alexanderwu -@File : logs.py +@File : llm.py """ -import sys +from metagpt.provider.anthropic_api import Claude2 as Claude +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM -from loguru import logger as _logger +DEFAULT_LLM = LLM() +CLAUDE_LLM = Claude() -from metagpt.const import PROJECT_ROOT - -def define_log_level(print_level="INFO", logfile_level="DEBUG"): - """Adjust log level to above the specified level.""" - _logger.remove() - _logger.add(sys.stderr, level=print_level) - _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) - return _logger - -logger = define_log_level() +async def ai_func(prompt): + """Use LLM for Q&A.""" + return await DEFAULT_LLM.aask(prompt) From 076ce7bee6dcd24cf9cf62d41663d75b1d0062ab Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:15:39 -0500 Subject: [PATCH 17/27] conflict resolutions --- .gitignore | 3 +- metagpt/actions/debug_error.py | 18 +----- metagpt/actions/run_code.py | 50 ++++++++++++++-- metagpt/actions/write_test.py | 104 +++++++++++++++++---------------- metagpt/roles/qa_engineer.py | 29 --------- metagpt/roles/role.py | 9 +-- metagpt/schema.py | 2 + metagpt/utils/common.py | 22 ++++++- 8 files changed, 132 insertions(+), 105 deletions(-) diff --git a/.gitignore b/.gitignore index 6abc90b43..6352a90e5 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,5 @@ examples/nb/ workspace/* *.mmd tmp -output.wav \ No newline at end of file +output.wav +metagpt/roles/idea_agent.py diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index c955f30ea..78a9e2409 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -27,21 +27,7 @@ Now you should start rewriting the code: class DebugError(Action): def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) - -<<<<<<< main - async def run(self, code, error): - prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ - f"\n\n{error}\n\nPlease try to fix the error in this code." - fixed_code = await self._aask(prompt) - return fixed_code - -======= - # async def run(self, code, error): - # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ - # f"\n\n{error}\n\nPlease try to fix the error in this code." - # fixed_code = await self._aask(prompt) - # return fixed_code - + async def run(self, context): if "PASS" in context: return "", "the original code works fine, no need to debug" @@ -57,4 +43,4 @@ class DebugError(Action): code = CodeParser.parse_code(block="", text=rsp) return file_name, code ->>>>>>> main + \ No newline at end of file diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 7fcf20975..0c8149c11 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -68,13 +68,55 @@ class RunCode(Action): return namespace.get('result', ""), "" except Exception: # If there is an error in the code, return the error message -<<<<<<< main - return traceback.format_exc() - -======= return "", traceback.format_exc() @classmethod async def run_script(cls, working_directory, additional_python_paths=[], command=[]) -> Tuple[str, str]: working_directory = str(working_directory) additional_python_paths = [str(path) for path in additional_python_paths] + # Copy the current environment variables + env = os.environ.copy() + + # Modify the PYTHONPATH environment variable + additional_python_paths = [working_directory] + additional_python_paths + additional_python_paths = ":".join(additional_python_paths) + env['PYTHONPATH'] = additional_python_paths + ':' + env.get('PYTHONPATH', '') + + # Start the subprocess + process = subprocess.Popen(command, cwd=working_directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + + try: + # Wait for the process to complete, with a timeout + stdout, stderr = process.communicate(timeout=10) + except subprocess.TimeoutExpired: + logger.info("The command did not complete within the given timeout.") + process.kill() # Kill the process if it times out + stdout, stderr = process.communicate() + return stdout.decode('utf-8'), stderr.decode('utf-8') + + async def run( + self, code, mode="script", code_file_name="", test_code="", test_file_name="", command=[], **kwargs + ) -> str: + logger.info(f"Running {' '.join(command)}") + if mode == "script": + outs, errs = await self.run_script(command=command, **kwargs) + elif mode == "text": + outs, errs = await self.run_text(code=code) + + logger.info(f"{outs=}") + logger.info(f"{errs=}") + + context = CONTEXT.format( + code=code, code_file_name=code_file_name, + test_code=test_code, test_file_name=test_file_name, + command=" ".join(command), + outs=outs[:500], # outs might be long but they are not important, truncate them to avoid token overflow + errs=errs[:10000] # truncate errors to avoid token overflow + ) + + prompt = PROMPT_TEMPLATE.format(context=context) + rsp = await self._aask(prompt) + + result = context + rsp + + return result \ No newline at end of file diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 73d4c119d..60fc92154 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -1,58 +1,64 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/11 17:45 +@Time : 2023/5/11 22:12 @Author : alexanderwu -@File : write_test.py +@File : environment.py """ -from metagpt.logs import logger -from metagpt.actions.action import Action -from metagpt.utils.common import CodeParser +import asyncio +from typing import Iterable -PROMPT_TEMPLATE = """ -NOTICE -1. Role: You are a QA engineer; the main goal is to design, develop, and execute PEP8 compliant, well-structured, maintainable test cases and scripts for Python 3.9. Your focus should be on ensuring the product quality of the entire project through systematic testing. -2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases. -3. Attention1: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script. -4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. -6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail? -7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE. -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. ------ -## Given the following code, please write appropriate test cases using Python's unittest framework to verify the correctness and robustness of this code: -```python -{code_to_test} -``` -Note that the code to test is at {source_file_path}, we will put your test code at {workspace}/tests/{test_file_name}, and run your test code from {workspace}, -you should correctly import the necessary classes based on these file locations! -## {test_file_name}: Write test code with triple quoto. Do your best to implement THIS ONLY ONE FILE. -""" +from pydantic import BaseModel, Field -class WriteTest(Action): - def __init__(self, name="WriteTest", context=None, llm=None): - super().__init__(name, context, llm) +from metagpt.memory import Memory +from metagpt.roles import Role +from metagpt.schema import Message - async def write_code(self, prompt): - code_rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=code_rsp) - return code -<<<<<<< main - async def run(self, code): - self.code = code - prompt = self.test_prompt_template.format(code=self.code) - test_cases = await self._aask(prompt) - return test_cases - -======= - async def run(self, code_to_test, test_file_name, source_file_path, workspace): - prompt = PROMPT_TEMPLATE.format( - code_to_test=code_to_test, - test_file_name=test_file_name, - source_file_path=source_file_path, - workspace=workspace - ) - code = await self.write_code(prompt) - return code ->>>>>>> main +class Environment(BaseModel): + """Environment that carries a set of roles. Roles can publish messages to the environment, which can be observed by other roles.""" + + roles: dict[str, Role] = Field(default_factory=dict) + memory: Memory = Field(default_factory=Memory) + history: str = Field(default='') + + class Config: + arbitrary_types_allowed = True + + def add_role(self, role: Role): + """Add a Role to the current environment.""" + role.set_env(self) + self.roles[role.profile] = role + + def add_roles(self, roles: Iterable[Role]): + """Add a batch of Roles to the current environment.""" + for role in roles: + self.add_role(role) + + def publish_message(self, message: Message): + """Publish a message to the current environment.""" + # self.message_queue.put(message) + self.memory.add(message) + self.history += f"\n{message}" + + async def run(self, k=1): + """Process the run of all Roles once.""" + # while not self.message_queue.empty(): + # message = self.message_queue.get() + # rsp = await self.manager.handle(message, self) + # self.message_queue.put(rsp) + for _ in range(k): + futures = [] + for role in self.roles.values(): + future = role.run() + futures.append(future) + + await asyncio.gather(*futures) + + def get_roles(self) -> dict[str, Role]: + """Get all Roles within the environment.""" + return self.roles + + def get_role(self, name: str) -> Role: + """Get a specified Role within the environment.""" + return self.roles.get(name, None) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index fe7f037b4..5e12a1abd 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -20,34 +20,6 @@ from metagpt.utils.common import CodeParser, parse_recipient from metagpt.utils.special_tokens import MSG_SEP, FILENAME_CODE_SEP class QaEngineer(Role): -<<<<<<< main - """ - Represents a Quality Assurance (QA) Engineer role responsible for writing tests to ensure software quality. - - Attributes: - name (str): Name of the QA engineer. - profile (str): Role profile. - goal (str): Goal of the QA engineer. - constraints (str): Constraints or limitations for the QA engineer. - """ - - def __init__(self, - name: str, - profile: str, - goal: str, - constraints: str) -> None: - """ - Initializes the QaEngineer role with given attributes. - - Args: - name (str): Name of the QA engineer. - profile (str): Role profile. - goal (str): Goal of the QA engineer. - constraints (str): Constraints or limitations for the QA engineer. - """ - super().__init__(name, profile, goal, constraints) - self._init_actions([WriteTest]) -======= def __init__(self, name="Edward", profile="QaEngineer", goal="Write comprehensive and robust tests to ensure codes will work as expected without bugs", constraints="The test code you write should conform to code standard like PEP8, be modular, easy to read and maintain", @@ -192,4 +164,3 @@ class QaEngineer(Role): role=self.profile, cause_by=WriteTest, sent_from=self.profile, send_to="" ) return result_msg ->>>>>>> main diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index fa2971ae7..b1ae51cf5 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -70,6 +70,7 @@ class RoleContext(BaseModel): state: int = Field(default=0) todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) + news: list[Type[Message]] = Field(default=[]) class Config: arbitrary_types_allowed = True @@ -183,16 +184,16 @@ class Role: env_msgs = self._rc.env.memory.get() observed = self._rc.env.memory.get_by_actions(self._rc.watch) - - news = self._rc.memory.remember(observed) # remember recent exact or similar memories + + self._rc.news = self._rc.memory.remember(observed) # remember recent exact or similar memories for i in env_msgs: self.recv(i) - news_text = [f"{i.role}: {i.content[:20]}..." for i in news] + news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news] if news_text: logger.debug(f'{self._setting} observed: {news_text}') - return len(news) + return len(self._rc.news) def _publish_message(self, msg): """If the role belongs to env, then the role's messages will be broadcast to env""" diff --git a/metagpt/schema.py b/metagpt/schema.py index ebcd71d8a..8db2c4ac1 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -27,6 +27,8 @@ class Message: instruct_content: BaseModel = field(default=None) role: str = field(default='user') # system / user / assistant cause_by: Type["Action"] = field(default="") + sent_from: str = field(default="") + send_to: str = field(default="") def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index b61dbc2eb..f65b38366 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -6,6 +6,7 @@ @File : common.py """ import ast +import contextlib import inspect import os import re @@ -78,6 +79,23 @@ class OutputParser: else: tasks = text.split("\n") return tasks + + @staticmethod + def parse_python_code(text: str) -> str: + for pattern in ( + r'(.*?```python.*?\s+)?(?P.*)(```.*?)', + r'(.*?```python.*?\s+)?(?P.*)', + ): + match = re.search(pattern, text, re.DOTALL) + if not match: + continue + code = match.group("code") + if not code: + continue + with contextlib.suppress(Exception): + ast.parse(code) + return code + raise ValueError("Invalid python code") @classmethod def parse_data(cls, data): @@ -183,7 +201,7 @@ class CodeParser: def parse_file_list(cls, block: str, text: str, lang: str = "") -> list[str]: # Regular expression pattern to find the tasks list. code = cls.parse_code(block, text, lang) - print(code) + # print(code) pattern = r'\s*(.*=.*)?(\[.*\])' # Extract tasks list string using regex. @@ -229,4 +247,4 @@ def print_members(module, indent=0): elif inspect.isfunction(obj): print(f'{prefix}Function: {name}') elif inspect.ismethod(obj): - print(f'{prefix}Method: {name}') \ No newline at end of file + print(f'{prefix}Method: {name}') From 6eb4d26224f91b9cb839d63cd726114c8277eed0 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 12:25:54 -0500 Subject: [PATCH 18/27] conflicts resolved --- metagpt/actions/debug_error.py | 4 -- metagpt/actions/run_code.py | 101 ++++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 75375a4fd..6b9fc744b 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -43,8 +43,4 @@ class DebugError(Action): code = CodeParser.parse_code(block="", text=rsp) return file_name, code -<<<<<<< HEAD -======= - ->>>>>>> 57ffca3b5c4559ac28b257ad5cb45f584769440d diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 24e33c89a..f40460371 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -18,6 +18,104 @@ from metagpt.schema import Message class Environment(BaseModel): """Environment that carries a set of roles. Roles can publish messages to the environment, which can be observed by other roles.""" + roles: dict[str, Role] = Field(default_factory=dict) + memory: Memory = Field(default_factory=Memory) + history: str = Field(default='') + + class Config: + arbitrary_types_allowed = True + + def add_role(self, role: Role): + """Add a Role to the current environment.""" + role.set_env(self) + self.roles[role.profile] = role + + def add_roles(self, roles: Iterable[Role]): + """Add a batch of Roles to the current environment.""" + for role in roles: + self.add_role(role) + + def publish_message(self, message: Message): + """Publish a message to the current environment.""" + # self.message_queue.put(message) + self.memory.add(message) + self.history += f"\n{message}" + + async def run(self, k=1): + """Process the run of all Roles once.""" + # while not self.message_queue.empty(): + # message = self.message_queue.get() + # rsp = await self.manager.handle(message, self) + # self.message_queue.put(rsp) + for _ in range(k): + futures = [] + for role in self.roles.values(): + future = role.run() + futures.append(future) + + await asyncio.gather(*futures) + + def get_roles(self) -> dict[str, Role]: + """Get all Roles within the environment.""" + return self.roles + + def get_role(self, name: str) -> Role: + """Get a specified Role within the environment.""" + return self.roles.get(name, None) +import traceback +import os +import subprocess +from typing import List, Tuple + +from metagpt.logs import logger +from metagpt.actions.action import Action + +PROMPT_TEMPLATE = """ +Role: You are a senior development and qa engineer, your role is summarize the code running result. +If the running result does not include an error, you should explicitly approve the result. +On the other hand, if the running result indicates some error, you should point out which part, the development code or the test code, produces the error, +and give specific instructions on fixing the errors. Here is the code info: +{context} +Now you should begin your analysis +--- +## instruction: +Please summarize the cause of the errors and give correction instruction +## File To Rewrite: +Determine the ONE file to rewrite in order to fix the error, for example, xyz.py, or test_xyz.py +## Status: +Determine if all of the code works fine, if so write PASS, else FAIL, +WRITE ONLY ONE WORD, PASS OR FAIL, IN THI SECTION +## Send To: +Please write Engineer if the errors are due to problematic development codes, and QaEngineer to problematic test codes, and NoOne if there are no errors, +WRITE ONLY ONE WORD, Engineer OR QaEngineer OR NoOne, IN THIS SECTION. +--- +You should fill in necessary instruction, status, send to, and finally return all content between the --- segment line. +""" + +CONTEXT = """ +## Development Code File Name +{code_file_name} +## Development Code +```python +{code} +``` +## Test File Name +{test_file_name} +## Test Code +```python +{test_code} +``` +## Running Command +{command} +## Running Output +standard output: {outs}; +standard errors: {errs}; +""" + +class RunCode(Action): + def __init__(self, name="RunCode", context=None, llm=None): + super().__init__(name, context, llm) + @classmethod async def run_text(cls, code) -> Tuple[str, str]: try: @@ -33,7 +131,8 @@ class Environment(BaseModel): async def run_script(cls, working_directory, additional_python_paths=[], command=[]) -> Tuple[str, str]: working_directory = str(working_directory) additional_python_paths = [str(path) for path in additional_python_paths] - # Copy the current environment variables + + # Copy the current environment variables env = os.environ.copy() # Modify the PYTHONPATH environment variable From 0ad84a4d42bf40ea154455930ccb52f8a58ba167 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 14:47:28 -0500 Subject: [PATCH 19/27] more confilt resolutions --- metagpt/actions/debug_error.py | 21 +++++++- metagpt/actions/write_test.py | 91 ++++++++++++++-------------------- metagpt/environment.py | 31 +++++++++--- metagpt/llm.py | 4 +- metagpt/logs.py | 4 +- metagpt/manager.py | 6 ++- metagpt/schema.py | 12 +++-- 7 files changed, 100 insertions(+), 69 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 6b9fc744b..114c414e4 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -42,5 +42,24 @@ class DebugError(Action): code = CodeParser.parse_code(block="", text=rsp) - return file_name, code + # async def run(self, code, error): + # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ + # f"\n\n{error}\n\nPlease try to fix the error in this code." + # fixed_code = await self._aask(prompt) + # return fixed_code + async def run(self, context): + if "PASS" in context: + return "", "the original code works fine, no need to debug" + + file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1) + + logger.info(f"Debug and rewrite {file_name}") + + prompt = PROMPT_TEMPLATE.format(context=context) + + rsp = await self._aask(prompt) + + code = CodeParser.parse_code(block="", text=rsp) + + return file_name, code diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 60fc92154..944d02192 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -5,60 +5,45 @@ @Author : alexanderwu @File : environment.py """ -import asyncio -from typing import Iterable +from metagpt.logs import logger +from metagpt.actions.action import Action +from metagpt.utils.common import CodeParser -from pydantic import BaseModel, Field +PROMPT_TEMPLATE = """ +NOTICE +1. Role: You are a QA engineer; the main goal is to design, develop, and execute PEP8 compliant, well-structured, maintainable test cases and scripts for Python 3.9. Your focus should be on ensuring the product quality of the entire project through systematic testing. +2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases. +3. Attention1: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script. +4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. +6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail? +7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE. +Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. +----- +## Given the following code, please write appropriate test cases using Python's unittest framework to verify the correctness and robustness of this code: +```python +{code_to_test} +``` +Note that the code to test is at {source_file_path}, we will put your test code at {workspace}/tests/{test_file_name}, and run your test code from {workspace}, +you should correctly import the necessary classes based on these file locations! +## {test_file_name}: Write test code with triple quoto. Do your best to implement THIS ONLY ONE FILE. +""" -from metagpt.memory import Memory -from metagpt.roles import Role -from metagpt.schema import Message +class WriteTest(Action): + def __init__(self, name="WriteTest", context=None, llm=None): + super().__init__(name, context, llm) + async def write_code(self, prompt): + code_rsp = await self._aask(prompt) + code = CodeParser.parse_code(block="", text=code_rsp) + return code -class Environment(BaseModel): - """Environment that carries a set of roles. Roles can publish messages to the environment, which can be observed by other roles.""" - - roles: dict[str, Role] = Field(default_factory=dict) - memory: Memory = Field(default_factory=Memory) - history: str = Field(default='') - - class Config: - arbitrary_types_allowed = True - - def add_role(self, role: Role): - """Add a Role to the current environment.""" - role.set_env(self) - self.roles[role.profile] = role - - def add_roles(self, roles: Iterable[Role]): - """Add a batch of Roles to the current environment.""" - for role in roles: - self.add_role(role) - - def publish_message(self, message: Message): - """Publish a message to the current environment.""" - # self.message_queue.put(message) - self.memory.add(message) - self.history += f"\n{message}" - - async def run(self, k=1): - """Process the run of all Roles once.""" - # while not self.message_queue.empty(): - # message = self.message_queue.get() - # rsp = await self.manager.handle(message, self) - # self.message_queue.put(rsp) - for _ in range(k): - futures = [] - for role in self.roles.values(): - future = role.run() - futures.append(future) - - await asyncio.gather(*futures) - - def get_roles(self) -> dict[str, Role]: - """Get all Roles within the environment.""" - return self.roles - - def get_role(self, name: str) -> Role: - """Get a specified Role within the environment.""" - return self.roles.get(name, None) + async def run(self, code_to_test, test_file_name, source_file_path, workspace): + prompt = PROMPT_TEMPLATE.format( + code_to_test=code_to_test, + test_file_name=test_file_name, + source_file_path=source_file_path, + workspace=workspace + ) + code = await self.write_code(prompt) + return code diff --git a/metagpt/environment.py b/metagpt/environment.py index 60fc92154..24e6ada2f 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -16,7 +16,10 @@ from metagpt.schema import Message class Environment(BaseModel): - """Environment that carries a set of roles. Roles can publish messages to the environment, which can be observed by other roles.""" + """环境,承载一批角色,角色可以向环境发布消息,可以被其他角色观察到 + Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles + + """ roles: dict[str, Role] = Field(default_factory=dict) memory: Memory = Field(default_factory=Memory) @@ -26,23 +29,31 @@ class Environment(BaseModel): arbitrary_types_allowed = True def add_role(self, role: Role): - """Add a Role to the current environment.""" + """增加一个在当前环境的角色 + Add a role in the current environment + """ role.set_env(self) self.roles[role.profile] = role def add_roles(self, roles: Iterable[Role]): - """Add a batch of Roles to the current environment.""" + """增加一批在当前环境的角色 + Add a batch of characters in the current environment + """ for role in roles: self.add_role(role) def publish_message(self, message: Message): - """Publish a message to the current environment.""" - # self.message_queue.put(message) + """向当前环境发布信息 + Post information to the current environment + """ + # self.message_queue.put(message) self.memory.add(message) self.history += f"\n{message}" async def run(self, k=1): - """Process the run of all Roles once.""" + """处理一次所有信息的运行 + Process all Role runs at once + """ # while not self.message_queue.empty(): # message = self.message_queue.get() # rsp = await self.manager.handle(message, self) @@ -56,9 +67,13 @@ class Environment(BaseModel): await asyncio.gather(*futures) def get_roles(self) -> dict[str, Role]: - """Get all Roles within the environment.""" + """获得环境内的所有角色 + Process all Role runs at once + """ return self.roles def get_role(self, name: str) -> Role: - """Get a specified Role within the environment.""" + """获得环境内的指定角色 + get all the environment roles + """ return self.roles.get(name, None) diff --git a/metagpt/llm.py b/metagpt/llm.py index b8aefec61..e6f815950 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -13,5 +13,7 @@ DEFAULT_LLM = LLM() CLAUDE_LLM = Claude() async def ai_func(prompt): - """Use LLM for Q&A.""" + """使用LLM进行QA + QA with LLMs + """ return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/logs.py b/metagpt/logs.py index c6755919d..b2052e9b8 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -13,7 +13,9 @@ from loguru import logger as _logger from metagpt.const import PROJECT_ROOT def define_log_level(print_level="INFO", logfile_level="DEBUG"): - """Adjust log level to above the specified level.""" + """调整日志级别到level之上 + Adjust the log level to above level + """ _logger.remove() _logger.add(sys.stderr, level=print_level) _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) diff --git a/metagpt/manager.py b/metagpt/manager.py index 5e5b256c0..9d238c621 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -32,7 +32,8 @@ class Manager: async def handle(self, message: Message, environment): """ - Manager handles the message, currently simply passes the message to the next person. + 管理员处理信息,现在简单的将信息递交给下一个人 + The administrator processes the information, now simply passes the information on to the next person :param message: :param environment: :return: @@ -49,7 +50,8 @@ class Manager: # Ask the LLM to decide which role should handle the message # chosen_role_name = self.llm.ask(self.prompt_template.format(context)) - # FIXME: Currently deciding the direction using a simple dictionary, but in the future, a thought process should be involved. + # FIXME: 现在通过简单的字典决定流向,但之后还是应该有思考过程 + #The direction of flow is now determined by a simple dictionary, but there should still be a thought process afterwards next_role_profile = self.role_directions[message.role] # logger.debug(f"{next_role_profile}") for _, role in roles.items(): diff --git a/metagpt/schema.py b/metagpt/schema.py index 8db2c4ac1..27f5dd10c 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -46,21 +46,27 @@ class Message: @dataclass class UserMessage(Message): - """Facilitates support for OpenAI messages""" + """便于支持OpenAI的消息 + Facilitate support for OpenAI messages + """ def __init__(self, content: str): super().__init__(content, 'user') @dataclass class SystemMessage(Message): - """Facilitates support for OpenAI messages""" + """便于支持OpenAI的消息 + Facilitate support for OpenAI messages + """ def __init__(self, content: str): super().__init__(content, 'system') @dataclass class AIMessage(Message): - """Facilitates support for OpenAI messages""" + """便于支持OpenAI的消息 + Facilitate support for OpenAI messages + """ def __init__(self, content: str): super().__init__(content, 'assistant') From 69e15ee6c057839f674b0d9e0194d701a77f0a35 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 14:48:44 -0500 Subject: [PATCH 20/27] final conflict resolved --- metagpt/actions/debug_error.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 114c414e4..22b1be6a4 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -48,6 +48,26 @@ class DebugError(Action): # fixed_code = await self._aask(prompt) # return fixed_code + async def run(self, context): + if "PASS" in context: + return "", "the original code works fine, no need to debug" + + file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1) + + logger.info(f"Debug and rewrite {file_name}") + + prompt = PROMPT_TEMPLATE.format(context=context) + + rsp = await self._aask(prompt) + + code = CodeParser.parse_code(block="", text=rsp) + + # async def run(self, code, error): + # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ + # f"\n\n{error}\n\nPlease try to fix the error in this code." + # fixed_code = await self._aask(prompt) + # return fixed_code + async def run(self, context): if "PASS" in context: return "", "the original code works fine, no need to debug" From b6a501d03dca79dda7ec6e89d7e5a8cb5b7f7e2d Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 14:53:30 -0500 Subject: [PATCH 21/27] fix of debug_error.py file --- metagpt/actions/debug_error.py | 36 +--------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 22b1be6a4..565826c33 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -27,20 +27,6 @@ Now you should start rewriting the code: class DebugError(Action): def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) - - async def run(self, context): - if "PASS" in context: - return "", "the original code works fine, no need to debug" - - file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1) - - logger.info(f"Debug and rewrite {file_name}") - - prompt = PROMPT_TEMPLATE.format(context=context) - - rsp = await self._aask(prompt) - - code = CodeParser.parse_code(block="", text=rsp) # async def run(self, code, error): # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ @@ -62,24 +48,4 @@ class DebugError(Action): code = CodeParser.parse_code(block="", text=rsp) - # async def run(self, code, error): - # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ - # f"\n\n{error}\n\nPlease try to fix the error in this code." - # fixed_code = await self._aask(prompt) - # return fixed_code - - async def run(self, context): - if "PASS" in context: - return "", "the original code works fine, no need to debug" - - file_name = re.search("## File To Rewrite:\s*(.+\\.py)", context).group(1) - - logger.info(f"Debug and rewrite {file_name}") - - prompt = PROMPT_TEMPLATE.format(context=context) - - rsp = await self._aask(prompt) - - code = CodeParser.parse_code(block="", text=rsp) - - return file_name, code + return file_name, code \ No newline at end of file From 91f6a29d72b2b27bf476f4f2711a79f3a9e52877 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 14:54:35 -0500 Subject: [PATCH 22/27] final conflict error --- metagpt/actions/run_code.py | 63 ++----------------------------------- 1 file changed, 3 insertions(+), 60 deletions(-) diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index f40460371..34700eb93 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -1,67 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/11 22:12 +@Time : 2023/5/11 17:46 @Author : alexanderwu -@File : environment.py +@File : run_code.py """ -import asyncio -from typing import Iterable - -from pydantic import BaseModel, Field - -from metagpt.memory import Memory -from metagpt.roles import Role -from metagpt.schema import Message - - -class Environment(BaseModel): - """Environment that carries a set of roles. Roles can publish messages to the environment, which can be observed by other roles.""" - - roles: dict[str, Role] = Field(default_factory=dict) - memory: Memory = Field(default_factory=Memory) - history: str = Field(default='') - - class Config: - arbitrary_types_allowed = True - - def add_role(self, role: Role): - """Add a Role to the current environment.""" - role.set_env(self) - self.roles[role.profile] = role - - def add_roles(self, roles: Iterable[Role]): - """Add a batch of Roles to the current environment.""" - for role in roles: - self.add_role(role) - - def publish_message(self, message: Message): - """Publish a message to the current environment.""" - # self.message_queue.put(message) - self.memory.add(message) - self.history += f"\n{message}" - - async def run(self, k=1): - """Process the run of all Roles once.""" - # while not self.message_queue.empty(): - # message = self.message_queue.get() - # rsp = await self.manager.handle(message, self) - # self.message_queue.put(rsp) - for _ in range(k): - futures = [] - for role in self.roles.values(): - future = role.run() - futures.append(future) - - await asyncio.gather(*futures) - - def get_roles(self) -> dict[str, Role]: - """Get all Roles within the environment.""" - return self.roles - - def get_role(self, name: str) -> Role: - """Get a specified Role within the environment.""" - return self.roles.get(name, None) import traceback import os import subprocess @@ -177,4 +120,4 @@ class RunCode(Action): result = context + rsp - return result + return result \ No newline at end of file From 13c05aa31f1c403cf351ceeeb7ada8d0b7fcd08a Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 14:57:53 -0500 Subject: [PATCH 23/27] final line for both files fixed --- metagpt/actions/debug_error.py | 2 +- metagpt/actions/run_code.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 565826c33..d69a22dba 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -48,4 +48,4 @@ class DebugError(Action): code = CodeParser.parse_code(block="", text=rsp) - return file_name, code \ No newline at end of file + return file_name, code diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 34700eb93..1bc5cc13a 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -120,4 +120,4 @@ class RunCode(Action): result = context + rsp - return result \ No newline at end of file + return result From 41bceb792ad9ec00f02e96a866383f3ae9a27f12 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Wed, 9 Aug 2023 21:47:36 -0500 Subject: [PATCH 24/27] fixed common.py error, missing last few lines --- metagpt/utils/common.py | 46 +++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index f65b38366..c3ac21df8 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -16,9 +16,9 @@ from metagpt.logs import logger def check_cmd_exists(command) -> int: - """ Check if the command exists - :param command: Command to be checked - :return: Returns 0 if the command exists, non-zero otherwise + """ 检查命令是否存在 + :param command: 待检查的命令 + :return: 如果命令存在,返回0,如果不存在,返回非0 """ check_command = 'command -v ' + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }' result = os.system(check_command) @@ -29,19 +29,19 @@ class OutputParser: @classmethod def parse_blocks(cls, text: str): - # First, split the text into different blocks based on "##" + # 首先根据"##"将文本分割成不同的block blocks = text.split("##") - # Create a dictionary to store the title and content of each block + # 创建一个字典,用于存储每个block的标题和内容 block_dict = {} - # Iterate through all blocks + # 遍历所有的block for block in blocks: - # If the block is not empty, continue processing + # 如果block不为空,则继续处理 if block.strip() != "": - # Separate the title and content of the block and trim whitespace + # 将block的标题和内容分开,并分别去掉前后的空白字符 block_title, block_content = block.split("\n", 1) - # LLM might have an error, correct it here + # LLM可能出错,在这里做一下修正 if block_title[-1] == ":": block_title = block_title[:-1] block_dict[block_title.strip()] = block_content.strip() @@ -102,13 +102,13 @@ class OutputParser: block_dict = cls.parse_blocks(data) parsed_data = {} for block, content in block_dict.items(): - # Try to remove the code marker + # 尝试去除code标记 try: content = cls.parse_code(text=content) except Exception: pass - # Try to parse the list + # 尝试解析list try: content = cls.parse_file_list(text=content) except Exception: @@ -121,7 +121,7 @@ class OutputParser: block_dict = cls.parse_blocks(data) parsed_data = {} for block, content in block_dict.items(): - # Try to remove the code marker + # 尝试去除code标记 try: content = cls.parse_code(text=content) except Exception: @@ -132,14 +132,14 @@ class OutputParser: else: typing = typing_define if typing == List[str] or typing == List[Tuple[str, str]]: - # Try to parse the list + # 尝试解析list try: content = cls.parse_file_list(text=content) except Exception: pass - # TODO: Removing extra quotes is risky, will address later + # TODO: 多余的引号去除有风险,后期再解决 # elif typing == str: - # # Try to remove extra quotes + # # 尝试去除多余的引号 # try: # content = cls.parse_str(text=content) # except Exception: @@ -160,17 +160,17 @@ class CodeParser: @classmethod def parse_blocks(cls, text: str): - # First, split the text into different blocks based on "##" + # 首先根据"##"将文本分割成不同的block blocks = text.split("##") - # Create a dictionary to store the title and content of each block + # 创建一个字典,用于存储每个block的标题和内容 block_dict = {} - # Iterate through all blocks + # 遍历所有的block for block in blocks: - # If the block is not empty, continue processing + # 如果block不为空,则继续处理 if block.strip() != "": - # Separate the title and content of the block and trim whitespace + # 将block的标题和内容分开,并分别去掉前后的空白字符 block_title, block_content = block.split("\n", 1) block_dict[block_title.strip()] = block_content.strip() @@ -248,3 +248,9 @@ def print_members(module, indent=0): print(f'{prefix}Function: {name}') elif inspect.ismethod(obj): print(f'{prefix}Method: {name}') + + +def parse_recipient(text): + pattern = r"## Send To:\s*([A-Za-z]+)\s*?" # hard code for now + recipient = re.search(pattern, text) + return recipient.group(1) if recipient else "" \ No newline at end of file From 193178b7d1616cb383e24292a5045040ca0c42cc Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Tue, 15 Aug 2023 06:51:39 -0500 Subject: [PATCH 25/27] resolved current conflicts --- Dockerfile | 16 +- metagpt/actions/__init__.py | 13 +- metagpt/config.py | 20 +-- metagpt/provider/openai_api.py | 79 ++++++--- metagpt/tools/sd_engine.py | 98 ++++++------ metagpt/tools/search_engine.py | 160 +++++++------------ metagpt/tools/search_engine_serpapi.py | 35 ++-- metagpt/tools/search_engine_serper.py | 27 ++-- metagpt/tools/web_browser_engine.py | 36 ++--- metagpt/tools/web_browser_engine_selenium.py | 33 ++-- metagpt/utils/mermaid.py | 42 +++-- 11 files changed, 290 insertions(+), 269 deletions(-) diff --git a/Dockerfile b/Dockerfile index be37f1df6..537bbc72e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # Use a base image with Python3.9 and Nodejs20 slim version FROM nikolaik/python-nodejs:python3.9-nodejs20-slim -# Install Debian software needed by MetaGPT +# Install Debian software needed by MetaGPT and clean up in one RUN command to reduce image size RUN apt update &&\ apt install -y git chromium fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 --no-install-recommends &&\ - apt clean + apt clean && rm -rf /var/lib/apt/lists/* # Install Mermaid CLI globally ENV CHROME_BIN="/usr/bin/chromium" \ @@ -15,13 +15,11 @@ RUN npm install -g @mermaid-js/mermaid-cli &&\ # Install Python dependencies and install MetaGPT COPY . /app/metagpt -RUN cd /app/metagpt &&\ - mkdir workspace &&\ - pip install -r requirements.txt &&\ - pip cache purge &&\ +WORKDIR /app/metagpt +RUN mkdir workspace &&\ + pip install --no-cache-dir -r requirements.txt &&\ python setup.py install -WORKDIR /app/metagpt - # Running with an infinite loop using the tail command -CMD ["sh", "-c", "tail -f /dev/null"] \ No newline at end of file +CMD ["sh", "-c", "tail -f /dev/null"] + diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index 165349728..b004bd58e 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -15,6 +15,7 @@ from metagpt.actions.design_api import WriteDesign from metagpt.actions.design_api_review import DesignReview from metagpt.actions.design_filenames import DesignFilenames from metagpt.actions.project_management import AssignTasks, WriteTasks +from metagpt.actions.research import CollectLinks, WebBrowseAndSummarize, ConductResearch from metagpt.actions.run_code import RunCode from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.actions.write_code import WriteCode @@ -26,6 +27,7 @@ from metagpt.actions.write_test import WriteTest class ActionType(Enum): """All types of Actions, used for indexing.""" + ADD_REQUIREMENT = BossRequirement WRITE_PRD = WritePRD WRITE_PRD_REVIEW = WritePRDReview @@ -40,4 +42,13 @@ class ActionType(Enum): WRITE_TASKS = WriteTasks ASSIGN_TASKS = AssignTasks SEARCH_AND_SUMMARIZE = SearchAndSummarize - \ No newline at end of file + COLLECT_LINKS = CollectLinks + WEB_BROWSE_AND_SUMMARIZE = WebBrowseAndSummarize + CONDUCT_RESEARCH = ConductResearch + + +__all__ = [ + "ActionType", + "Action", + "ActionOutput", +] diff --git a/metagpt/config.py b/metagpt/config.py index 3753bb3b0..faeffd777 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -4,14 +4,14 @@ Provide configuration, singleton """ import os -import openai +import openai import yaml from metagpt.const import PROJECT_ROOT from metagpt.logs import logger -from metagpt.utils.singleton import Singleton from metagpt.tools import SearchEngineType, WebBrowserEngineType +from metagpt.utils.singleton import Singleton class NotConfiguredException(Exception): @@ -46,7 +46,6 @@ class Config(metaclass=Singleton): self.openai_api_key = self._get("OPENAI_API_KEY") if not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key: raise NotConfiguredException("Set OPENAI_API_KEY first") - self.openai_api_base = self._get("OPENAI_API_BASE") if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base: openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy @@ -67,21 +66,22 @@ class Config(metaclass=Singleton): self.google_api_key = self._get("GOOGLE_API_KEY") self.google_cse_id = self._get("GOOGLE_CSE_ID") self.search_engine = self._get("SEARCH_ENGINE", SearchEngineType.SERPAPI_GOOGLE) - + self.web_browser_engine = WebBrowserEngineType(self._get("WEB_BROWSER_ENGINE", "playwright")) self.playwright_browser_type = self._get("PLAYWRIGHT_BROWSER_TYPE", "chromium") self.selenium_browser_type = self._get("SELENIUM_BROWSER_TYPE", "chrome") - + self.long_term_memory = self._get('LONG_TERM_MEMORY', False) if self.long_term_memory: logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 - self.puppeteer_config = self._get("PUPPETEER_CONFIG","") - self.mmdc = self._get("MMDC","mmdc") - self.update_costs = self._get("UPDATE_COSTS",True) - self.calc_usage = self._get("CALC_USAGE",True) - + self.puppeteer_config = self._get("PUPPETEER_CONFIG", "") + self.mmdc = self._get("MMDC", "mmdc") + self.update_costs = self._get("UPDATE_COSTS", True) + self.calc_usage = self._get("CALC_USAGE", True) + self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") + self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 6f7c33c4f..86b63770c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @Time : 2023/5/5 23:08 @@ -7,10 +6,11 @@ """ import asyncio import time -from functools import wraps from typing import NamedTuple import openai +from openai.error import APIConnectionError +from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type from metagpt.config import CONFIG from metagpt.logs import logger @@ -20,8 +20,10 @@ from metagpt.utils.token_counter import ( TOKEN_COSTS, count_message_tokens, count_string_tokens, + get_max_completion_tokens, ) +<<<<<<< main def retry(max_retries): def decorator(f): @wraps(f) @@ -41,10 +43,21 @@ class RateLimiter: def __init__(self, rpm): self.last_call_time = 0 self.interval = 1.1 * 60 / rpm # Here 1.1 is used because even if the calls are made strictly on time, they will still be QOS'd; consider switching to simple error retry later +======= + +class RateLimiter: + """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed""" + + def __init__(self, rpm): + self.last_call_time = 0 + # Here 1.1 is used because even if the calls are made strictly according to time, + # they will still be QOS'd; consider switching to simple error retry later + self.interval = 1.1 * 60 / rpm +>>>>>>> main self.rpm = rpm def split_batches(self, batch): - return [batch[i:i + self.rpm] for i in range(0, len(batch), self.rpm)] + return [batch[i : i + self.rpm] for i in range(0, len(batch), self.rpm)] async def wait_if_needed(self, num_requests): current_time = time.time() @@ -64,7 +77,8 @@ class Costs(NamedTuple): total_budget: float class CostManager(metaclass=Singleton): - """Calculate the cost of using the interface""" + """计算使用接口的开销""" + def __init__(self): self.total_prompt_tokens = 0 self.total_completion_tokens = 0 @@ -82,13 +96,12 @@ class CostManager(metaclass=Singleton): """ self.total_prompt_tokens += prompt_tokens self.total_completion_tokens += completion_tokens - cost = ( - prompt_tokens * TOKEN_COSTS[model]["prompt"] - + completion_tokens * TOKEN_COSTS[model]["completion"] - ) / 1000 + cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"]) / 1000 self.total_cost += cost - logger.info(f"Total running cost: ${self.total_cost:.3f} | Max budget: ${CONFIG.max_budget:.3f} | " - f"Current cost: ${cost:.3f}, {prompt_tokens=}, {completion_tokens=}") + logger.info( + f"Total running cost: ${self.total_cost:.3f} | Max budget: ${CONFIG.max_budget:.3f} | " + f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}" + ) CONFIG.total_cost = self.total_cost def get_total_prompt_tokens(self): @@ -122,14 +135,25 @@ def get_costs(self) -> Costs: """Get all costs""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) +def log_and_reraise(retry_state): + logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") + logger.warning(""" +Recommend going to https://deepwisdom.feishu.cn/wiki/MsGnwQBjiif9c3koSJNcYaoSnu4#part-XdatdVlhEojeAfxaaEZcMV3ZniQ +See FAQ 5.8 +""") + raise retry_state.outcome.exception() + + class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ Check https://platform.openai.com/examples for examples """ + def __init__(self): self.__init_openai(CONFIG) self.llm = openai self.model = CONFIG.openai_api_model + self.auto_max_tokens = False self._cost_manager = CostManager() RateLimiter.__init__(self, rpm=self.rpm) @@ -143,10 +167,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: - response = await openai.ChatCompletion.acreate( - **self._cons_kwargs(messages), - stream=True - ) + response = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages), stream=True) # create variables to collect the stream of chunks collected_chunks = [] @@ -154,41 +175,42 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # iterate through the stream of events async for chunk in response: collected_chunks.append(chunk) # save the event response - chunk_message = chunk['choices'][0]['delta'] # extract the message + chunk_message = chunk["choices"][0]["delta"] # extract the message collected_messages.append(chunk_message) # save the message if "content" in chunk_message: print(chunk_message["content"], end="") print() - full_reply_content = ''.join([m.get('content', '') for m in collected_messages]) + full_reply_content = "".join([m.get("content", "") for m in collected_messages]) usage = self._calc_usage(messages, full_reply_content) self._update_costs(usage) return full_reply_content def _cons_kwargs(self, messages: list[dict]) -> dict: - if CONFIG.openai_api_type == 'azure': + if CONFIG.openai_api_type == "azure": kwargs = { "deployment_id": CONFIG.deployment_id, "messages": messages, - "max_tokens": CONFIG.max_tokens_rsp, + "max_tokens": self.get_max_tokens(messages), "n": 1, "stop": None, - "temperature": 0.3 + "temperature": 0.3, } else: kwargs = { "model": self.model, "messages": messages, - "max_tokens": CONFIG.max_tokens_rsp, + "max_tokens": self.get_max_tokens(messages), "n": 1, "stop": None, - "temperature": 0.3 + "temperature": 0.3, } + kwargs["timeout"] = 3 return kwargs async def _achat_completion(self, messages: list[dict]) -> dict: rsp = await self.llm.ChatCompletion.acreate(**self._cons_kwargs(messages)) - self._update_costs(rsp.get('usage')) + self._update_costs(rsp.get("usage")) return rsp def _chat_completion(self, messages: list[dict]) -> dict: @@ -206,7 +228,13 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # messages = self.messages_to_dict(messages) return await self._achat_completion(messages) - @retry(max_retries=6) + @retry( + stop=stop_after_attempt(3), + wait=wait_fixed(1), + after=after_log(logger, logger.level('WARNING').name), + retry=retry_if_exception_type(APIConnectionError), + retry_error_callback=log_and_reraise, + ) async def acompletion_text(self, messages: list[dict], stream=False) -> str: """when streaming, print each token in place.""" if stream: @@ -257,3 +285,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def get_costs(self) -> Costs: return self._cost_manager.get_costs() + + def get_max_tokens(self, messages: list[dict]): + if not self.auto_max_tokens: + return CONFIG.max_tokens_rsp + return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index e212c2fc7..1d9cd0b2a 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -2,29 +2,27 @@ # @Date : 2023/7/19 16:28 # @Author : stellahong (stellahong@fuzhi.ai) # @Desc : -import os import asyncio +import base64 +import io +import json +import os from os.path import join from typing import List -import json -import io -import base64 from aiohttp import ClientSession from PIL import Image, PngImagePlugin -from metagpt.logs import logger from metagpt.config import Config from metagpt.const import WORKSPACE_ROOT +from metagpt.logs import logger config = Config() payload = { "prompt": "", "negative_prompt": "(easynegative:0.8),black, dark,Low resolution", - "override_settings": { - "sd_model_checkpoint": "galaxytimemachinesGTM_photoV20" - }, + "override_settings": {"sd_model_checkpoint": "galaxytimemachinesGTM_photoV20"}, "seed": -1, "batch_size": 1, "n_iter": 1, @@ -36,21 +34,20 @@ payload = { "tiling": False, "do_not_save_samples": False, "do_not_save_grid": False, - 'enable_hr': False, - 'hr_scale': 2, - 'hr_upscaler': 'Latent', - 'hr_second_pass_steps': 0, - 'hr_resize_x': 0, - 'hr_resize_y': 0, - 'hr_upscale_to_x': 0, - 'hr_upscale_to_y': 0, - 'truncate_x': 0, - 'truncate_y': 0, - 'applied_old_hires_behavior_to': None, + "enable_hr": False, + "hr_scale": 2, + "hr_upscaler": "Latent", + "hr_second_pass_steps": 0, + "hr_resize_x": 0, + "hr_resize_y": 0, + "hr_upscale_to_x": 0, + "hr_upscale_to_y": 0, + "truncate_x": 0, + "truncate_y": 0, + "applied_old_hires_behavior_to": None, "eta": None, - "sampler_index": "DPM++ SDE Karras", - "alwayson_scripts": {} + "alwayson_scripts": {}, } default_negative_prompt = "(easynegative:0.8),black, dark,Low resolution" @@ -60,14 +57,20 @@ class SDEngine: def __init__(self): # Initialize the SDEngine with configuration self.config = Config() - self.sd_url = self.config.get('SD_URL') + self.sd_url = self.config.get("SD_URL") self.sd_t2i_url = f"{self.sd_url}{self.config.get('SD_T2I_API')}" # Define default payload settings for SD API self.payload = payload logger.info(self.sd_t2i_url) - - def construct_payload(self, prompt, negtive_prompt=default_negative_prompt, width=512, height=512, - sd_model="galaxytimemachinesGTM_photoV20"): + + def construct_payload( + self, + prompt, + negtive_prompt=default_negative_prompt, + width=512, + height=512, + sd_model="galaxytimemachinesGTM_photoV20", + ): # Configure the payload with provided inputs self.payload["prompt"] = prompt self.payload["negtive_prompt"] = negtive_prompt @@ -76,13 +79,13 @@ class SDEngine: self.payload["override_settings"]["sd_model_checkpoint"] = sd_model logger.info(f"call sd payload is {self.payload}") return self.payload - + def _save(self, imgs, save_name=""): - save_dir = WORKSPACE_ROOT / "resources"/"SD_Output" + save_dir = WORKSPACE_ROOT / "resources" / "SD_Output" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) - + async def run_t2i(self, prompts: List): # Asynchronously run the SD API for multiple prompts session = ClientSession() @@ -90,24 +93,25 @@ class SDEngine: results = await self.run(url=self.sd_t2i_url, payload=payload, session=session) self._save(results, save_name=f"output_{payload_idx}") await session.close() - -async def run(self, url, payload, session): - # Perform the HTTP POST request to the SD API - async with session.post(url, json=payload, timeout=600) as rsp: - data = await rsp.read() - - rsp_json = json.loads(data) - imgs = rsp_json['images'] - logger.info(f"callback rsp json is {rsp_json.keys()}") - return imgs -async def run_i2i(self): - # todo: Add image-to-image interface call - raise NotImplementedError + async def run(self, url, payload, session): + # Perform the HTTP POST request to the SD API + async with session.post(url, json=payload, timeout=600) as rsp: + data = await rsp.read() + + rsp_json = json.loads(data) + imgs = rsp_json["images"] + logger.info(f"callback rsp json is {rsp_json.keys()}") + return imgs + + async def run_i2i(self): + # todo: 添加图生图接口调用 + raise NotImplementedError + + async def run_sam(self): + # todo:添加SAM接口调用 + raise NotImplementedError -async def run_sam(self): - # todo: Add SAM interface call - raise NotImplementedError def decode_base64_to_image(img, save_name): image = Image.open(io.BytesIO(base64.b64decode(img.split(",", 1)[0]))) @@ -122,12 +126,10 @@ def batch_decode_base64_to_image(imgs, save_dir="", save_name=""): decode_base64_to_image(_img, save_name=save_name) if __name__ == "__main__": - import asyncio - engine = SDEngine() prompt = "pixel style, game design, a game interface should be minimalistic and intuitive with the score and high score displayed at the top. The snake and its food should be easily distinguishable. The game should have a simple color scheme, with a contrasting color for the snake and its food. Complete interface boundary" - + engine.construct_payload(prompt) - + event_loop = asyncio.get_event_loop() event_loop.run_until_complete(engine.run_t2i(prompt)) diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 1668dfb5c..d28700054 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -7,118 +7,76 @@ """ from __future__ import annotations -import json +import importlib +from typing import Callable, Coroutine, Literal, overload -from metagpt.config import Config -from metagpt.logs import logger -from metagpt.tools.search_engine_serpapi import SerpAPIWrapper -from metagpt.tools.search_engine_serper import SerperWrapper - -config = Config() +from metagpt.config import CONFIG from metagpt.tools import SearchEngineType class SearchEngine: - """ - TODO: Integrate Google Search and reverse proxy. - Note: Google here requires a Proxifier or similar global proxy. - - DDG: https://pypi.org/project/duckduckgo-search/ - - GOOGLE: https://programmablesearchengine.google.com/controlpanel/overview?cx=63f9de531d0e24de9 - """ - def __init__(self, engine=None, run_func=None): - self.config = Config() - self.run_func = run_func - self.engine = engine or self.config.search_engine + """Class representing a search engine. - @classmethod - def run_google(cls, query, max_results=8): - # results = ddg(query, max_results=max_results) - results = google_official_search(query, num_results=max_results) - logger.info(results) - return results + Args: + engine: The search engine type. Defaults to the search engine specified in the config. + run_func: The function to run the search. Defaults to None. - async def run(self, query: str, max_results=8): - if self.engine == SearchEngineType.SERPAPI_GOOGLE: - api = SerpAPIWrapper() - rsp = await api.run(query) - elif self.engine == SearchEngineType.DIRECT_GOOGLE: - rsp = SearchEngine.run_google(query, max_results) - elif self.engine == SearchEngineType.SERPER_GOOGLE: - api = SerperWrapper() - rsp = await api.run(query) - elif self.engine == SearchEngineType.CUSTOM_ENGINE: - rsp = self.run_func(query) + Attributes: + run_func: The function to run the search. + engine: The search engine type. + """ + def __init__( + self, + engine: SearchEngineType | None = None, + run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None, + ): + engine = engine or CONFIG.search_engine + if engine == SearchEngineType.SERPAPI_GOOGLE: + module = "metagpt.tools.search_engine_serpapi" + run_func = importlib.import_module(module).SerpAPIWrapper().run + elif engine == SearchEngineType.SERPER_GOOGLE: + module = "metagpt.tools.search_engine_serper" + run_func = importlib.import_module(module).SerperWrapper().run + elif engine == SearchEngineType.DIRECT_GOOGLE: + module = "metagpt.tools.search_engine_googleapi" + run_func = importlib.import_module(module).GoogleAPIWrapper().run + elif engine == SearchEngineType.DUCK_DUCK_GO: + module = "metagpt.tools.search_engine_ddg" + run_func = importlib.import_module(module).DDGAPIWrapper().run + elif engine == SearchEngineType.CUSTOM_ENGINE: + pass # run_func = run_func else: raise NotImplementedError - return rsp + self.engine = engine + self.run_func = run_func -def google_official_search(query: str, num_results: int = 8, focus=['snippet', 'link', 'title']) -> dict | list[dict]: - """Return the results of a Google search using the official Google API + @overload + def run( + self, + query: str, + max_results: int = 8, + as_string: Literal[True] = True, + ) -> str: + ... - Args: - query (str): The search query. - num_results (int): The number of results to return. + @overload + def run( + self, + query: str, + max_results: int = 8, + as_string: Literal[False] = False, + ) -> list[dict[str, str]]: + ... - Returns: - str: The results of the search. - """ + async def run(self, query: str, max_results: int = 8, as_string: bool = True) -> str | list[dict[str, str]]: + """Run a search query. - from googleapiclient.discovery import build - from googleapiclient.errors import HttpError + Args: + query: The search query. + max_results: The maximum number of results to return. Defaults to 8. + as_string: Whether to return the results as a string or a list of dictionaries. Defaults to True. - try: - api_key = config.google_api_key - custom_search_engine_id = config.google_cse_id - - with build("customsearch", "v1", developerKey=api_key) as service: - - result = ( - service.cse() - .list(q=query, cx=custom_search_engine_id, num=num_results) - .execute() - ) - logger.info(result) - # Extract the search result items from the response - search_results = result.get("items", []) - - # Create a list of only the URLs from the search results - search_results_details = [{i: j for i, j in item_dict.items() if i in focus} for item_dict in search_results] - - except HttpError as e: - # Handle errors in the API call - error_details = json.loads(e.content.decode()) - - # Check if the error is related to an invalid or missing API key - if error_details.get("error", {}).get( - "code" - ) == 403 and "invalid API key" in error_details.get("error", {}).get( - "message", "" - ): - return "Error: The provided Google API key is invalid or missing." - else: - return f"Error: {e}" - - # Return the list of search result URLs - return search_results_details - -def safe_google_results(results: str | list) -> str: - """ - Return the results of a google search in a safe format. - - Args: - results (str | list): The search results. - - Returns: - str: The results of the search. - """ - if isinstance(results, list): - safe_message = json.dumps( - [result for result in results] - ) - else: - safe_message = results.encode("utf-8", "ignore").decode("utf-8") - return safe_message - -if __name__ == '__main__': - SearchEngine.run(query='wtf') - \ No newline at end of file + Returns: + The search results as a string or a list of dictionaries. + """ + return await self.run_func(query, max_results=max_results, as_string=as_string) diff --git a/metagpt/tools/search_engine_serpapi.py b/metagpt/tools/search_engine_serpapi.py index 2bf07b342..3d2d7cfe4 100644 --- a/metagpt/tools/search_engine_serpapi.py +++ b/metagpt/tools/search_engine_serpapi.py @@ -37,16 +37,17 @@ class SerpAPIWrapper(BaseModel): class Config: arbitrary_types_allowed = True - async def run(self, query: str, **kwargs: Any) -> str: + async def run(self, query: str, max_results: int = 8, as_string: bool = True, **kwargs: Any) -> str: """Run query through SerpAPI and parse result async.""" - return self._process_response(await self.results(query)) + return self._process_response(await self.results(query, max_results), as_string=as_string) - async def results(self, query: str) -> dict: + async def results(self, query: str, max_results: int) -> dict: """Use aiohttp to run query through SerpAPI and return the results async.""" def construct_url_and_params() -> Tuple[str, Dict[str, str]]: params = self.get_params(query) params["source"] = "python" + params["num"] = max_results if self.serpapi_api_key: params["serp_api_key"] = self.serpapi_api_key params["output"] = "json" @@ -74,10 +75,10 @@ class SerpAPIWrapper(BaseModel): return params @staticmethod - def _process_response(res: dict) -> str: + def _process_response(res: dict, as_string: bool) -> str: """Process response from SerpAPI.""" # logger.debug(res) - focus = ['title', 'snippet', 'link'] + focus = ["title", "snippet", "link"] get_focused = lambda x: {i: j for i, j in x.items() if i in focus} if "error" in res.keys(): @@ -86,20 +87,11 @@ class SerpAPIWrapper(BaseModel): toret = res["answer_box"]["answer"] elif "answer_box" in res.keys() and "snippet" in res["answer_box"].keys(): toret = res["answer_box"]["snippet"] - elif ( - "answer_box" in res.keys() - and "snippet_highlighted_words" in res["answer_box"].keys() - ): + elif "answer_box" in res.keys() and "snippet_highlighted_words" in res["answer_box"].keys(): toret = res["answer_box"]["snippet_highlighted_words"][0] - elif ( - "sports_results" in res.keys() - and "game_spotlight" in res["sports_results"].keys() - ): + elif "sports_results" in res.keys() and "game_spotlight" in res["sports_results"].keys(): toret = res["sports_results"]["game_spotlight"] - elif ( - "knowledge_graph" in res.keys() - and "description" in res["knowledge_graph"].keys() - ): + elif "knowledge_graph" in res.keys() and "description" in res["knowledge_graph"].keys(): toret = res["knowledge_graph"]["description"] elif "snippet" in res["organic_results"][0].keys(): toret = res["organic_results"][0]["snippet"] @@ -112,5 +104,10 @@ class SerpAPIWrapper(BaseModel): if res.get("organic_results"): toret_l += [get_focused(i) for i in res.get("organic_results")] - return str(toret) + '\n' + str(toret_l) - \ No newline at end of file + return str(toret) + '\n' + str(toret_l) if as_string else toret_l + + +if __name__ == "__main__": + import fire + + fire.Fire(SerpAPIWrapper().run) diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index 45c19090c..2ae2c3b7d 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -36,16 +36,19 @@ class SerperWrapper(BaseModel): class Config: arbitrary_types_allowed = True - async def run(self, query: str, **kwargs: Any) -> str: + async def run(self, query: str, max_results: int = 8, as_string: bool = True, **kwargs: Any) -> str: """Run query through Serper and parse result async.""" - queries = query.split("\n") - return "\n".join([self._process_response(res) for res in await self.results(queries)]) + if isinstance(query, str): + return self._process_response((await self.results([query], max_results))[0], as_string=as_string) + else: + results = [self._process_response(res, as_string) for res in await self.results(query, max_results)] + return "\n".join(results) if as_string else results - async def results(self, queries: list[str]) -> dict: + async def results(self, queries: list[str], max_results: int = 8) -> dict: """Use aiohttp to run query through Serper and return the results async.""" def construct_url_and_payload_and_headers() -> Tuple[str, Dict[str, str]]: - payloads = self.get_payloads(queries) + payloads = self.get_payloads(queries, max_results) url = "https://google.serper.dev/search" headers = self.get_headers() return url, payloads, headers @@ -61,12 +64,13 @@ class SerperWrapper(BaseModel): return res - def get_payloads(self, queries: list[str]) -> Dict[str, str]: + def get_payloads(self, queries: list[str], max_results: int) -> Dict[str, str]: """Get payloads for Serper.""" payloads = [] for query in queries: _payload = { "q": query, + "num": max_results, } payloads.append({**self.payload, **_payload}) return json.dumps(payloads, sort_keys=True) @@ -79,7 +83,7 @@ class SerperWrapper(BaseModel): return headers @staticmethod - def _process_response(res: dict) -> str: + def _process_response(res: dict, as_string: bool = False) -> str: """Process response from SerpAPI.""" # logger.debug(res) focus = ['title', 'snippet', 'link'] @@ -117,5 +121,10 @@ class SerperWrapper(BaseModel): if res.get("organic"): toret_l += [get_focused(i) for i in res.get("organic")] - return str(toret) + '\n' + str(toret_l) - \ No newline at end of file + return str(toret) + '\n' + str(toret_l) if as_string else toret_l + + +if __name__ == "__main__": + import fire + + fire.Fire(SerperWrapper().run) diff --git a/metagpt/tools/web_browser_engine.py b/metagpt/tools/web_browser_engine.py index 67b794dd1..453d87f31 100644 --- a/metagpt/tools/web_browser_engine.py +++ b/metagpt/tools/web_browser_engine.py @@ -1,22 +1,20 @@ #!/usr/bin/env python from __future__ import annotations -import asyncio -import importlib -from typing import Any, Callable, Coroutine, overload +import importlib +from typing import Any, Callable, Coroutine, Literal, overload from metagpt.config import CONFIG from metagpt.tools import WebBrowserEngineType -from bs4 import BeautifulSoup +from metagpt.utils.parse_html import WebPage class WebBrowserEngine: def __init__( self, engine: WebBrowserEngineType | None = None, - run_func: Callable[..., Coroutine[Any, Any, str | list[str]]] | None = None, - parse_func: Callable[[str], str] | None = None, + run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, ): engine = engine or CONFIG.web_browser_engine @@ -30,31 +28,25 @@ class WebBrowserEngine: run_func = run_func else: raise NotImplementedError - self.parse_func = parse_func or get_page_content self.run_func = run_func self.engine = engine @overload - async def run(self, url: str) -> str: + async def run(self, url: str) -> WebPage: ... @overload - async def run(self, url: str, *urls: str) -> list[str]: + async def run(self, url: str, *urls: str) -> list[WebPage]: ... - async def run(self, url: str, *urls: str) -> str | list[str]: - page = await self.run_func(url, *urls) - if isinstance(page, str): - return self.parse_func(page) - return [self.parse_func(i) for i in page] - - -def get_page_content(page: str): - soup = BeautifulSoup(page, "html.parser") - return "\n".join(i.text.strip() for i in soup.find_all(["h1", "h2", "h3", "h4", "h5", "p", "pre"])) + async def run(self, url: str, *urls: str) -> WebPage | list[WebPage]: + return await self.run_func(url, *urls) if __name__ == "__main__": - text = asyncio.run(WebBrowserEngine().run("https://fuzhi.ai/")) - print(text) - \ No newline at end of file + import fire + + async def main(url: str, *urls: str, engine_type: Literal["playwright", "selenium"] = "playwright", **kwargs): + return await WebBrowserEngine(WebBrowserEngineType(engine_type), **kwargs).run(url, *urls) + + fire.Fire(main) diff --git a/metagpt/tools/web_browser_engine_selenium.py b/metagpt/tools/web_browser_engine_selenium.py index 78533e05a..d727709b8 100644 --- a/metagpt/tools/web_browser_engine_selenium.py +++ b/metagpt/tools/web_browser_engine_selenium.py @@ -2,16 +2,17 @@ from __future__ import annotations import asyncio -from copy import deepcopy import importlib +from concurrent import futures +from copy import deepcopy from typing import Literal -from metagpt.config import CONFIG -import asyncio from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait -from concurrent import futures + +from metagpt.config import CONFIG +from metagpt.utils.parse_html import WebPage class SeleniumWrapper: @@ -48,7 +49,7 @@ class SeleniumWrapper: self.loop = loop self.executor = executor - async def run(self, url: str, *urls: str) -> str | list[str]: + async def run(self, url: str, *urls: str) -> WebPage | list[WebPage]: await self._run_precheck() _scrape = lambda url: self.loop.run_in_executor(self.executor, self._scrape_website, url) @@ -69,9 +70,15 @@ class SeleniumWrapper: def _scrape_website(self, url): with self._get_driver() as driver: - driver.get(url) - WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.TAG_NAME, "body"))) - return driver.page_source + try: + driver.get(url) + WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.TAG_NAME, "body"))) + inner_text = driver.execute_script("return document.body.innerText;") + html = driver.page_source + except Exception as e: + inner_text = f"Fail to load page content for {e}" + html = "" + return WebPage(inner_text=inner_text, html=html, url=url) _webdriver_manager_types = { @@ -97,6 +104,7 @@ def _gen_get_driver_func(browser_type, *args, executable_path=None): def _get_driver(): options = Options() options.add_argument("--headless") + options.add_argument("--enable-javascript") if browser_type == "chrome": options.add_argument("--no-sandbox") for i in args: @@ -107,6 +115,9 @@ def _gen_get_driver_func(browser_type, *args, executable_path=None): if __name__ == "__main__": - text = asyncio.run(SeleniumWrapper("chrome").run("https://fuzhi.ai/")) - print(text) - \ No newline at end of file + import fire + + async def main(url: str, *urls: str, browser_type: str = "chrome", **kwargs): + return await SeleniumWrapper(browser_type, **kwargs).run(url, *urls) + + fire.Fire(main) diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 17ac0db4a..24aabe8ae 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -5,9 +5,9 @@ @Author : alexanderwu @File : mermaid.py """ -import os import subprocess from pathlib import Path + from metagpt.config import CONFIG from metagpt.const import PROJECT_ROOT from metagpt.logs import logger @@ -24,25 +24,36 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height :return: 0 if succed, -1 if failed """ # Write the Mermaid code to a temporary file - tmp = Path(f'{output_file_without_suffix}.mmd') - tmp.write_text(mermaid_code, encoding='utf-8') + tmp = Path(f"{output_file_without_suffix}.mmd") + tmp.write_text(mermaid_code, encoding="utf-8") - if check_cmd_exists('mmdc') != 0: - logger.warning( - "RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") + if check_cmd_exists("mmdc") != 0: + logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") return -1 - for suffix in ['pdf', 'svg', 'png']: - output_file = f'{output_file_without_suffix}.{suffix}' + for suffix in ["pdf", "svg", "png"]: + output_file = f"{output_file_without_suffix}.{suffix}" # Call the `mmdc` command to convert the Mermaid code to a PNG logger.info(f"Generating {output_file}..") if CONFIG.puppeteer_config: - subprocess.run([CONFIG.mmdc, '-p', CONFIG.puppeteer_config, '-i', str(tmp), '-o', - output_file, '-w', str(width), '-H', str(height)]) + subprocess.run( + [ + CONFIG.mmdc, + "-p", + CONFIG.puppeteer_config, + "-i", + str(tmp), + "-o", + output_file, + "-w", + str(width), + "-H", + str(height), + ] + ) else: - subprocess.run([CONFIG.mmdc, '-i', str(tmp), '-o', - output_file, '-w', str(width), '-H', str(height)]) + subprocess.run([CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) return 0 @@ -97,8 +108,7 @@ MMC2 = """sequenceDiagram SE-->>M: return summary""" -if __name__ == '__main__': +if __name__ == "__main__": # logger.info(print_members(print_members)) - mermaid_to_file(MMC1, PROJECT_ROOT / 'tmp/1.png') - mermaid_to_file(MMC2, PROJECT_ROOT / 'tmp/2.png') - \ No newline at end of file + mermaid_to_file(MMC1, PROJECT_ROOT / "tmp/1.png") + mermaid_to_file(MMC2, PROJECT_ROOT / "tmp/2.png") From 3ce123a7bf29390b4b400e6b8679ec9c55c47bbe Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Tue, 15 Aug 2023 07:46:02 -0500 Subject: [PATCH 26/27] conflict fixes --- metagpt/config.py | 8 +- metagpt/provider/openai_api.py | 22 ---- .../tools/web_browser_engine_playwright.py | 102 +++++++++++------- 3 files changed, 69 insertions(+), 63 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index faeffd777..d47d5325b 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -76,10 +76,10 @@ class Config(metaclass=Singleton): logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 - self.puppeteer_config = self._get("PUPPETEER_CONFIG", "") - self.mmdc = self._get("MMDC", "mmdc") - self.update_costs = self._get("UPDATE_COSTS", True) - self.calc_usage = self._get("CALC_USAGE", True) + + self.puppeteer_config = self._get("PUPPETEER_CONFIG","") + self.mmdc = self._get("MMDC","mmdc") + self.calc_usage = self._get("CALC_USAGE",True) self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 86b63770c..551810546 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -23,27 +23,6 @@ from metagpt.utils.token_counter import ( get_max_completion_tokens, ) -<<<<<<< main -def retry(max_retries): - def decorator(f): - @wraps(f) - async def wrapper(*args, **kwargs): - for i in range(max_retries): - try: - return await f(*args, **kwargs) - except Exception: - if i == max_retries - 1: - raise - await asyncio.sleep(2 ** i) - return wrapper - return decorator - -class RateLimiter: - """Rate limiter class, each call goes through wait_if_needed, sleep if rate limiting is required""" - def __init__(self, rpm): - self.last_call_time = 0 - self.interval = 1.1 * 60 / rpm # Here 1.1 is used because even if the calls are made strictly on time, they will still be QOS'd; consider switching to simple error retry later -======= class RateLimiter: """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed""" @@ -53,7 +32,6 @@ class RateLimiter: # Here 1.1 is used because even if the calls are made strictly according to time, # they will still be QOS'd; consider switching to simple error retry later self.interval = 1.1 * 60 / rpm ->>>>>>> main self.rpm = rpm def split_batches(self, batch): diff --git a/metagpt/tools/web_browser_engine_playwright.py b/metagpt/tools/web_browser_engine_playwright.py index 94539e9a3..030e7701b 100644 --- a/metagpt/tools/web_browser_engine_playwright.py +++ b/metagpt/tools/web_browser_engine_playwright.py @@ -2,12 +2,15 @@ from __future__ import annotations import asyncio -from pathlib import Path import sys +from pathlib import Path from typing import Literal + from playwright.async_api import async_playwright + from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.utils.parse_html import WebPage class PlaywrightWrapper: @@ -16,7 +19,7 @@ class PlaywrightWrapper: To use this module, you should have the `playwright` Python package installed and ensure that the required browsers are also installed. You can install playwright by running the command `pip install metagpt[playwright]` and download the necessary browser binaries by running the - command `playwright install` for the first time." + command `playwright install` for the first time. """ def __init__( @@ -40,27 +43,30 @@ class PlaywrightWrapper: self._context_kwargs = context_kwargs self._has_run_precheck = False - async def run(self, url: str, *urls: str) -> str | list[str]: + async def run(self, url: str, *urls: str) -> WebPage | list[WebPage]: async with async_playwright() as ap: browser_type = getattr(ap, self.browser_type) await self._run_precheck(browser_type) browser = await browser_type.launch(**self.launch_kwargs) - - async def _scrape(url): - context = await browser.new_context(**self._context_kwargs) - page = await context.new_page() - async with page: - try: - await page.goto(url) - await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") - content = await page.content() - return content - except Exception as e: - return f"Fail to load page content for {e}" + _scrape = self._scrape if urls: - return await asyncio.gather(_scrape(url), *(_scrape(i) for i in urls)) - return await _scrape(url) + return await asyncio.gather(_scrape(browser, url), *(_scrape(browser, i) for i in urls)) + return await _scrape(browser, url) + + async def _scrape(self, browser, url): + context = await browser.new_context(**self._context_kwargs) + page = await context.new_page() + async with page: + try: + await page.goto(url) + await page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + html = await page.content() + inner_text = await page.evaluate("() => document.body.innerText") + except Exception as e: + inner_text = f"Fail to load page content for {e}" + html = "" + return WebPage(inner_text=inner_text, html=html, url=url) async def _run_precheck(self, browser_type): if self._has_run_precheck: @@ -72,6 +78,10 @@ class PlaywrightWrapper: if CONFIG.global_proxy: kwargs["env"] = {"ALL_PROXY": CONFIG.global_proxy} await _install_browsers(self.browser_type, **kwargs) + + if self._has_run_precheck: + return + if not executable_path.exists(): parts = executable_path.parts available_paths = list(Path(*parts[:-3]).glob(f"{self.browser_type}-*")) @@ -85,25 +95,37 @@ class PlaywrightWrapper: self._has_run_precheck = True +def _get_install_lock(): + global _install_lock + if _install_lock is None: + _install_lock = asyncio.Lock() + return _install_lock + + async def _install_browsers(*browsers, **kwargs) -> None: - process = await asyncio.create_subprocess_exec( - sys.executable, - "-m", - "playwright", - "install", - *browsers, - "--with-deps", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - **kwargs, - ) + async with _get_install_lock(): + browsers = [i for i in browsers if i not in _install_cache] + if not browsers: + return + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "playwright", + "install", + *browsers, + # "--with-deps", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + **kwargs, + ) - await asyncio.gather(_log_stream(process.stdout, logger.info), _log_stream(process.stderr, logger.warning)) + await asyncio.gather(_log_stream(process.stdout, logger.info), _log_stream(process.stderr, logger.warning)) - if await process.wait() == 0: - logger.info(f"Install browser for playwright successfully.") - else: - logger.warning(f"Fail to install browser for playwright.") + if await process.wait() == 0: + logger.info("Install browser for playwright successfully.") + else: + logger.warning("Fail to install browser for playwright.") + _install_cache.update(browsers) async def _log_stream(sr, log_func): @@ -114,8 +136,14 @@ async def _log_stream(sr, log_func): log_func(f"[playwright install browser]: {line.decode().strip()}") +_install_lock: asyncio.Lock = None +_install_cache = set() + + if __name__ == "__main__": - for i in ("chromium", "firefox", "webkit"): - text = asyncio.run(PlaywrightWrapper(i).run("https://httpbin.org/ip")) - print(text) - print(i) \ No newline at end of file + import fire + + async def main(url: str, *urls: str, browser_type: str = "chromium", **kwargs): + return await PlaywrightWrapper(browser_type, **kwargs).run(url, *urls) + + fire.Fire(main) From 21629f841b0a8357e8015831a7929434776a2800 Mon Sep 17 00:00:00 2001 From: brucemeek <113046530+brucemeek@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:01:53 -0500 Subject: [PATCH 27/27] config conflict fixed --- metagpt/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index d47d5325b..f4f3ebf7a 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -44,8 +44,10 @@ class Config(metaclass=Singleton): logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") - if not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key: - raise NotConfiguredException("Set OPENAI_API_KEY first") + self.anthropic_api_key = self._get("Anthropic_API_KEY") + if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) \ + and (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key): + raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base: openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy