From feef54ba3bcbd0d6fe7943f6e9aaf318b9cc396a Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Sun, 17 Dec 2023 13:52:37 +0800 Subject: [PATCH 01/49] patch release v0.5.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 730fffd35..73a05eeae 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ with open(path.join(here, "requirements.txt"), encoding="utf-8") as f: setup( name="metagpt", - version="0.5.0", + version="0.5.1", description="The Multi-Role Meta Programming Framework", long_description=long_description, long_description_content_type="text/markdown", From 097c6e09d2ff7b0f5487f6068ff2185801eb950e Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sun, 17 Dec 2023 14:41:59 +0800 Subject: [PATCH 02/49] add deprecated warnings for the start_project method --- metagpt/team.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/metagpt/team.py b/metagpt/team.py index a5c405f80..5ce07ef13 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -3,10 +3,11 @@ """ @Time : 2023/5/12 00:30 @Author : alexanderwu -@File : software_company.py +@File : team.py @Modified By: mashenquan, 2023/11/27. Add an archiving operation after completing the project, as specified in Section 2.2.3.3 of RFC 135. """ +import warnings from pydantic import BaseModel, Field from metagpt.actions import UserRequirement @@ -47,7 +48,7 @@ class Team(BaseModel): raise NoMoneyException(CONFIG.total_cost, f"Insufficient funds: {CONFIG.max_budget}") def run_project(self, idea, send_to: str = ""): - """Start a project from publishing user requirement.""" + """Run a project from publishing user requirement.""" self.idea = idea # Human requirement. @@ -55,6 +56,16 @@ class Team(BaseModel): Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL) ) + def start_project(self, idea, send_to: str = ""): + """ + Deprecated: This method will be removed in the future. + Please use the `run_project` method instead. + """ + warnings.warn("The 'start_project' method is deprecated and will be removed in the future. " + "Please use the 'run_project' method instead.", + DeprecationWarning, stacklevel=2) + return self.run_project(idea=idea, send_to=send_to) + def _save(self): logger.info(self.json(ensure_ascii=False)) From 43e35fe929d165f6f5c36b8f683d85825fa78684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 18 Dec 2023 16:13:21 +0800 Subject: [PATCH 03/49] fixbug: recursive user requirement dead loop --- metagpt/roles/role.py | 23 ++++++----------------- metagpt/schema.py | 4 ---- tests/metagpt/test_role.py | 6 +++--- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1e7ebf711..48688ad5f 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -25,9 +25,8 @@ from typing import Iterable, Set, Type from pydantic import BaseModel, Field -from metagpt.actions import Action, ActionOutput +from metagpt.actions import Action, ActionOutput, UserRequirement from metagpt.actions.action_node import ActionNode -from metagpt.actions.add_requirement import UserRequirement from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger from metagpt.memory import Memory @@ -127,17 +126,7 @@ class RoleContext(BaseModel): return self.memory.get() -class _RoleInjector(type): - def __call__(cls, *args, **kwargs): - instance = super().__call__(*args, **kwargs) - - if not instance._rc.watch: - instance._watch([UserRequirement]) - - return instance - - -class Role(metaclass=_RoleInjector): +class Role: """Role/Agent""" def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False): @@ -149,10 +138,9 @@ class Role(metaclass=_RoleInjector): self._states = [] self._actions = [] self._role_id = str(self._setting) - self._rc = RoleContext() + self._rc = RoleContext(watch={any_to_str(UserRequirement)}) self._subscription = {any_to_str(self), name} if name else {any_to_str(self)} - def _reset(self): self._states = [] self._actions = [] @@ -203,8 +191,7 @@ class Role(metaclass=_RoleInjector): """Watch Actions of interest. Role will select Messages caused by these Actions from its personal message buffer during _observe. """ - tags = {any_to_str(t) for t in actions} - self._rc.watch.update(tags) + self._rc.watch = {any_to_str(t) for t in actions} # check RoleContext after adding watch actions self._rc.check(self._role_id) @@ -401,6 +388,8 @@ class Role(metaclass=_RoleInjector): msg = with_message elif isinstance(with_message, list): msg = Message("\n".join(with_message)) + if not msg.cause_by: + msg.cause_by = UserRequirement self.put_message(msg) if not await self._observe(): diff --git a/metagpt/schema.py b/metagpt/schema.py index 5aec378e4..758149efa 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -121,10 +121,6 @@ class Message(BaseModel): :param send_to: Specifies the target recipient or consumer for message delivery in the environment. :param role: Message meta info tells who sent this message. """ - if not cause_by: - from metagpt.actions import UserRequirement - cause_by = UserRequirement - super().__init__( id=uuid.uuid4().hex, content=content, diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 8fac2503c..611d321fc 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -14,11 +14,11 @@ import uuid import pytest from pydantic import BaseModel -from metagpt.actions import Action, ActionOutput +from metagpt.actions import Action, ActionOutput, UserRequirement from metagpt.environment import Environment from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import get_class_name +from metagpt.utils.common import any_to_str, get_class_name class MockAction(Action): @@ -60,7 +60,7 @@ async def test_react(): name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, desc=seed.desc ) role.subscribe({seed.subscription}) - assert role._rc.watch == set({}) + assert role._rc.watch == {any_to_str(UserRequirement)} assert role.name == seed.name assert role.profile == seed.profile assert role._setting.goal == seed.goal From a88f931fe9a93dc7d883c0a260310f9aee9942e0 Mon Sep 17 00:00:00 2001 From: garylin2099 Date: Mon, 18 Dec 2023 19:26:38 +0800 Subject: [PATCH 04/49] update version and roadmap --- docs/ROADMAP.md | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index afc9ff445..3cb03f374 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -30,10 +30,10 @@ ### Tasks 4. Complete the design and implementation of module breakdown 5. Support various modes of memory: clearly distinguish between long-term and short-term memory 6. Perfect the test role, and carry out necessary interactions with humans - 7. Allowing natural communication between roles (expected v0.5.0) + 7. ~~Allowing natural communication between roles~~ (v0.5.0) 8. Implement SkillManager and the process of incremental Skill learning (experimentation done with game agents) 9. Automatically get RPM and configure it by calling the corresponding openai page, so that each key does not need to be manually configured - 10. IMPORTANT: Support incremental development (expected v0.5.0) + 10. ~~IMPORTANT: Support incremental development~~ (v0.5.0) 3. Strategies 1. Support ReAct strategy (experimentation done with game agents) 2. Support CoT strategy (experimentation done with game agents) @@ -45,8 +45,8 @@ ### Tasks 2. Implementation: Knowledge search, supporting 10+ data formats 3. Implementation: Data EDA (expected v0.6.0) 4. Implementation: Review - 5. Implementation: Add Document (expected v0.5.0) - 6. Implementation: Delete Document (expected v0.5.0) + 5. ~~Implementation~~: Add Document (v0.5.0) + 6. ~~Implementation~~: Delete Document (v0.5.0) 7. Implementation: Self-training 8. ~~Implementation: DebugError~~ (v0.2.1) 9. Implementation: Generate reliable unit tests based on YAPI diff --git a/setup.py b/setup.py index 73a05eeae..57290f4cd 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ with open(path.join(here, "requirements.txt"), encoding="utf-8") as f: setup( name="metagpt", - version="0.5.1", + version="0.5.2", description="The Multi-Role Meta Programming Framework", long_description=long_description, long_description_content_type="text/markdown", From 5022e2e713adaefe0cfce44939f918174026509a Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 10:52:16 +0800 Subject: [PATCH 05/49] remove requirements-ocr.txt and place the optional setup to setup.py --- requirements-ocr.txt | 4 ---- setup.py | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 requirements-ocr.txt diff --git a/requirements-ocr.txt b/requirements-ocr.txt deleted file mode 100644 index cf6103afc..000000000 --- a/requirements-ocr.txt +++ /dev/null @@ -1,4 +0,0 @@ -paddlepaddle==2.4.2 -paddleocr>=2.0.1 -tabulate==0.9.0 --r requirements.txt diff --git a/setup.py b/setup.py index 4dd453b3d..64d34f1e9 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ setup( "search-google": ["google-api-python-client==2.94.0"], "search-ddg": ["duckduckgo-search==3.8.5"], "pyppeteer": ["pyppeteer>=1.0.2"], + "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"], }, cmdclass={ "install_mermaid": InstallMermaidCLI, From caac36b83f795ba556e063574fa85a5cfea8fb3c Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 11:01:20 +0800 Subject: [PATCH 06/49] use pre-commit --- metagpt/actions/action_node.py | 12 ++++++++++-- metagpt/actions/project_management_an.py | 2 +- metagpt/actions/write_code_review.py | 8 ++++++-- metagpt/roles/architect.py | 2 +- metagpt/roles/engineer.py | 2 +- metagpt/roles/project_manager.py | 2 +- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index fb7d621d8..9bb12fc84 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -52,6 +52,7 @@ def dict_to_markdown(d, prefix="-", postfix="\n"): class ActionNode: """ActionNode is a tree of nodes.""" + mode: str # Action Context @@ -70,8 +71,15 @@ class ActionNode: content: str instruct_content: BaseModel - def __init__(self, key: str, expected_type: Type, instruction: str, example: str, content: str = "", - children: dict[str, "ActionNode"] = None): + def __init__( + self, + key: str, + expected_type: Type, + instruction: str, + example: str, + content: str = "", + children: dict[str, "ActionNode"] = None, + ): self.key = key self.expected_type = expected_type self.instruction = instruction diff --git a/metagpt/actions/project_management_an.py b/metagpt/actions/project_management_an.py index 970cb0594..6208c1051 100644 --- a/metagpt/actions/project_management_an.py +++ b/metagpt/actions/project_management_an.py @@ -44,7 +44,7 @@ FULL_API_SPEC = ActionNode( key="Full API spec", expected_type=str, instruction="Describe all APIs using OpenAPI 3.0 spec that may be used by both frontend and backend. If front-end " - "and back-end communication is not required, leave it blank.", + "and back-end communication is not required, leave it blank.", example="openapi: 3.0.0 ...", ) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 4b3e9aece..365c87063 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -154,11 +154,15 @@ class WriteCodeReview(Action): code=iterative_code, filename=self.context.code_doc.filename, ) - cr_prompt = EXAMPLE_AND_INSTRUCTION.format(format_example=format_example, ) + cr_prompt = EXAMPLE_AND_INSTRUCTION.format( + format_example=format_example, + ) logger.info( f"Code review and rewrite {self.context.code_doc.filename}: {i+1}/{k} | {len(iterative_code)=}, {len(self.context.code_doc.content)=}" ) - result, rewrited_code = await self.write_code_review_and_rewrite(context_prompt, cr_prompt, self.context.code_doc.filename) + result, rewrited_code = await self.write_code_review_and_rewrite( + context_prompt, cr_prompt, self.context.code_doc.filename + ) if "LBTM" in result: iterative_code = rewrited_code elif "LGTM" in result: diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index fa91d393d..fce6c3425 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -28,7 +28,7 @@ class Architect(Role): profile: str = "Architect", goal: str = "design a concise, usable, complete software system", constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries." - "Use same language as user requirement" + "Use same language as user requirement", ) -> None: """Initializes the Architect with given attributes.""" super().__init__(name, profile, goal, constraints) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index f1e65b177..2620fe4e5 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -73,7 +73,7 @@ class Engineer(Role): profile: str = "Engineer", goal: str = "write elegant, readable, extensible, efficient code", constraints: str = "the code should conform to standards like google-style and be modular and maintainable. " - "Use same language as user requirement", + "Use same language as user requirement", n_borg: int = 1, use_code_review: bool = False, ) -> None: diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 5a2b9be50..657737513 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -26,7 +26,7 @@ class ProjectManager(Role): name: str = "Eve", profile: str = "Project Manager", goal: str = "break down tasks according to PRD/technical design, generate a task list, and analyze task " - "dependencies to start with the prerequisite modules", + "dependencies to start with the prerequisite modules", constraints: str = "use same language as user requirement", ) -> None: """ From bef1071c5ab2928ac4df4448870e53d97206fc33 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 11:10:17 +0800 Subject: [PATCH 07/49] setup.py: update --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 64d34f1e9..320230a7f 100644 --- a/setup.py +++ b/setup.py @@ -30,15 +30,15 @@ with open(path.join(here, "requirements.txt"), encoding="utf-8") as f: setup( name="metagpt", - version="0.4.0", - description="The Multi-Role Meta Programming Framework", + version="0.5.1", + description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/geekan/MetaGPT", author="Alexander Wu", author_email="alexanderwu@deepwisdom.ai", license="MIT", - keywords="metagpt multi-role multi-agent programming gpt llm metaprogramming", + keywords="metagpt multi-agent multi-role programming gpt llm metaprogramming", packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, From 8b8ee5c56d778468141f4f6e007fcf9fb10e1bc7 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 11:22:21 +0800 Subject: [PATCH 08/49] delete inspect_module.py because we have ast tree parser --- metagpt/inspect_module.py | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 metagpt/inspect_module.py diff --git a/metagpt/inspect_module.py b/metagpt/inspect_module.py deleted file mode 100644 index 48ceffc57..000000000 --- a/metagpt/inspect_module.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/28 14:54 -@Author : alexanderwu -@File : inspect_module.py -""" - -import inspect - -import metagpt # replace with your module - - -def print_classes_and_functions(module): - """FIXME: NOT WORK..""" - for name, obj in inspect.getmembers(module): - if inspect.isclass(obj): - print(f"Class: {name}") - elif inspect.isfunction(obj): - print(f"Function: {name}") - else: - print(name) - - print(dir(module)) - - -if __name__ == "__main__": - print_classes_and_functions(metagpt) From ad8f7ebfb9c74cb8ef258563ad9c71666a178596 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 11:49:06 +0800 Subject: [PATCH 09/49] token_counter: add gpt-3.5-turbo-16k in list and add comment for them --- metagpt/utils/token_counter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 266a53268..ebfb85de7 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -56,6 +56,7 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): if model in { "gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", + "gpt-3.5-turbo-16k", "gpt-3.5-turbo-1106", "gpt-4-0314", "gpt-4-32k-0314", @@ -63,7 +64,7 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): "gpt-4-32k-0613", "gpt-4-1106-preview", }: - tokens_per_message = 3 + tokens_per_message = 3 # # every reply is primed with <|start|>assistant<|message|> tokens_per_name = 1 elif model == "gpt-3.5-turbo-0301": tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n From 602818e2baf9ffc4b2108d8a8c23fd7dc1a19522 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 11:52:23 +0800 Subject: [PATCH 10/49] openai_api: refine logic --- metagpt/provider/openai_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index a73bb0aa0..86054881e 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -329,7 +329,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): usage["completion_tokens"] = completion_tokens return usage except Exception as e: - logger.error("usage calculation failed!", e) + logger.error(f"{self.model} usage calculation failed!", e) + return {} else: return usage @@ -360,7 +361,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return results def _update_costs(self, usage: dict): - if CONFIG.calc_usage: + if CONFIG.calc_usage and usage: try: prompt_tokens = int(usage["prompt_tokens"]) completion_tokens = int(usage["completion_tokens"]) From e8f45c4072a6987975bed97375f66161e3273c43 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 13:40:54 +0800 Subject: [PATCH 11/49] delete utils.py, move function to common.py --- metagpt/actions/action.py | 3 +-- metagpt/utils/common.py | 18 ++++++++++++++++++ metagpt/utils/utils.py | 22 ---------------------- 3 files changed, 19 insertions(+), 24 deletions(-) delete mode 100644 metagpt/utils/utils.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 1534b1f4d..7bb26ea91 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -15,8 +15,7 @@ from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM from metagpt.logs import logger from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess -from metagpt.utils.common import OutputParser -from metagpt.utils.utils import general_after_log +from metagpt.utils.common import OutputParser, general_after_log class Action(ABC): diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index a9bdd6e2d..e9061d548 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -17,8 +17,11 @@ import inspect import os import platform import re +import typing from typing import List, Tuple, Union +from tenacity import _utils + from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.logs import logger @@ -363,3 +366,18 @@ def is_subscribed(message, tags): if t in message.send_to: return True return False + + +def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: + def log_it(retry_state: "RetryCallState") -> None: + if retry_state.fn is None: + fn_name = "" + else: + fn_name = _utils.get_callback_name(retry_state.fn) + logger.error( + f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " + f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " + f"exp: {retry_state.outcome.exception()}" + ) + + return log_it diff --git a/metagpt/utils/utils.py b/metagpt/utils/utils.py deleted file mode 100644 index 5ceed65d9..000000000 --- a/metagpt/utils/utils.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Desc : - -import typing - -from tenacity import _utils - - -def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: - def log_it(retry_state: "RetryCallState") -> None: - if retry_state.fn is None: - fn_name = "" - else: - fn_name = _utils.get_callback_name(retry_state.fn) - logger.error( - f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " - f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " - f"exp: {retry_state.outcome.exception()}" - ) - - return log_it From 6f166603c4e574d6983d5a2879cc3ad3a539b2ab Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 13:51:51 +0800 Subject: [PATCH 12/49] add function import, avoid "import" --- metagpt/utils/common.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index e9061d548..d0528544b 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -20,7 +20,8 @@ import re import typing from typing import List, Tuple, Union -from tenacity import _utils +import loguru +from tenacity import RetryCallState, _utils from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.logs import logger From d3c135edff1e5e9d34fc8414d84d4e34e3963054 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 14:17:54 +0800 Subject: [PATCH 13/49] refine utils code --- metagpt/utils/common.py | 51 ++++++++++++++++++++++++------------ tests/metagpt/test_role.py | 8 +++--- tests/metagpt/test_schema.py | 8 +++--- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index d0528544b..cdabe96a3 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -295,9 +295,6 @@ class NoMoneyException(Exception): def print_members(module, indent=0): """ https://stackoverflow.com/questions/1796180/how-can-i-get-a-list-of-all-classes-within-current-module-in-python - :param module: - :param indent: - :return: """ prefix = " " * indent for name, obj in inspect.getmembers(module): @@ -315,6 +312,7 @@ def print_members(module, indent=0): def parse_recipient(text): + # FIXME: use ActionNode instead. pattern = r"## Send To:\s*([A-Za-z]+)\s*?" # hard code for now recipient = re.search(pattern, text) if recipient: @@ -331,18 +329,12 @@ def get_class_name(cls) -> str: return f"{cls.__module__}.{cls.__name__}" -def get_object_name(obj) -> str: - """Return class name of the object""" - cls = type(obj) - return f"{cls.__module__}.{cls.__name__}" - - -def any_to_str(val) -> str: +def any_to_str(val: str | typing.Callable) -> str: """Return the class name or the class name of the object, or 'val' if it's a string type.""" if isinstance(val, str): return val if not callable(val): - return get_object_name(val) + return get_class_name(type(val)) return get_class_name(val) @@ -350,32 +342,57 @@ def any_to_str(val) -> str: def any_to_str_set(val) -> set: """Convert any type to string set.""" res = set() - if isinstance(val, dict) or isinstance(val, list) or isinstance(val, set) or isinstance(val, tuple): + + # Check if the value is iterable, but not a string (since strings are technically iterable) + if isinstance(val, (dict, list, set, tuple)): + # Special handling for dictionaries to iterate over values + if isinstance(val, dict): + val = val.values() + for i in val: res.add(any_to_str(i)) else: res.add(any_to_str(val)) + return res -def is_subscribed(message, tags): +def is_subscribed(message: "Message", tags: set): """Return whether it's consumer""" if MESSAGE_ROUTE_TO_ALL in message.send_to: return True - for t in tags: - if t in message.send_to: + for i in tags: + if i in message.send_to: return True return False -def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: +def general_after_log(i: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: + """ + Generates a logging function to be used after a call is retried. + + This generated function logs an error message with the outcome of the retried function call. It includes + the name of the function, the time taken for the call in seconds (formatted according to `sec_format`), + the number of attempts made, and the exception raised, if any. + + :param i: A Logger instance from the loguru library used to log the error message. + :param sec_format: A string format specifier for how to format the number of seconds since the start of the call. + Defaults to three decimal places. + :return: A callable that accepts a RetryCallState object and returns None. This callable logs the details + of the retried call. + """ + def log_it(retry_state: "RetryCallState") -> None: + # If the function name is not known, default to "" if retry_state.fn is None: fn_name = "" else: + # Retrieve the callable's name using a utility function fn_name = _utils.get_callback_name(retry_state.fn) - logger.error( + + # Log an error message with the function name, time since start, attempt number, and the exception + i.error( f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " f"exp: {retry_state.outcome.exception()}" diff --git a/tests/metagpt/test_role.py b/tests/metagpt/test_role.py index 8fac2503c..3328607cf 100644 --- a/tests/metagpt/test_role.py +++ b/tests/metagpt/test_role.py @@ -18,7 +18,7 @@ from metagpt.actions import Action, ActionOutput from metagpt.environment import Environment from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import get_class_name +from metagpt.utils.common import any_to_str class MockAction(Action): @@ -88,13 +88,13 @@ async def test_react(): @pytest.mark.asyncio async def test_msg_to(): m = Message(content="a", send_to=["a", MockRole, Message]) - assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) + assert m.send_to == {"a", any_to_str(MockRole), any_to_str(Message)} m = Message(content="a", cause_by=MockAction, send_to={"a", MockRole, Message}) - assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) + assert m.send_to == {"a", any_to_str(MockRole), any_to_str(Message)} m = Message(content="a", send_to=("a", MockRole, Message)) - assert m.send_to == set({"a", get_class_name(MockRole), get_class_name(Message)}) + assert m.send_to == {"a", any_to_str(MockRole), any_to_str(Message)} if __name__ == "__main__": diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 51ebd5baa..79421fab2 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -13,7 +13,7 @@ import pytest from metagpt.actions import Action from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage -from metagpt.utils.common import get_class_name +from metagpt.utils.common import any_to_str @pytest.mark.asyncio @@ -54,9 +54,9 @@ def test_message(): m.cause_by = "Message" assert m.cause_by == "Message" m.cause_by = Action - assert m.cause_by == get_class_name(Action) + assert m.cause_by == any_to_str(Action) m.cause_by = Action() - assert m.cause_by == get_class_name(Action) + assert m.cause_by == any_to_str(Action) m.content = "b" assert m.content == "b" @@ -67,7 +67,7 @@ def test_routes(): m.send_to = "b" assert m.send_to == {"b"} m.send_to = {"e", Action} - assert m.send_to == {"e", get_class_name(Action)} + assert m.send_to == {"e", any_to_str(Action)} if __name__ == "__main__": From f1c6a7ebfbad1c571574ca4f9b85c14e24221e33 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 16:16:52 +0800 Subject: [PATCH 14/49] refine code: use handle_exception function instead of in-function duplicate code frags --- metagpt/actions/action_node.py | 2 +- metagpt/actions/run_code.py | 30 ++++----- metagpt/config.py | 1 + metagpt/repo_parser.py | 19 ++++-- metagpt/schema.py | 78 ++++++++-------------- metagpt/tools/search_engine_meilisearch.py | 12 ++-- metagpt/utils/common.py | 10 +++ metagpt/utils/custom_decoder.py | 2 +- metagpt/utils/dependency_file.py | 20 ++---- metagpt/utils/exceptions.py | 59 ++++++++++++++++ metagpt/utils/file.py | 45 ++++++------- metagpt/utils/file_repository.py | 11 +-- 12 files changed, 159 insertions(+), 130 deletions(-) create mode 100644 metagpt/utils/exceptions.py diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 9bb12fc84..6f1215920 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -43,7 +43,7 @@ Fill in the above nodes based on the format example. """ -def dict_to_markdown(d, prefix="-", postfix="\n"): +def dict_to_markdown(d, prefix="###", postfix="\n"): markdown_str = "" for key, value in d.items(): markdown_str += f"{prefix} {key}: {value}{postfix}" diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index fa13a0980..1b9fd252f 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -16,13 +16,13 @@ class. """ import subprocess -import traceback from typing import Tuple from metagpt.actions.action import Action from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import RunCodeResult +from metagpt.utils.exceptions import handle_exception PROMPT_TEMPLATE = """ Role: You are a senior development and qa engineer, your role is summarize the code running result. @@ -78,15 +78,12 @@ class RunCode(Action): super().__init__(name, context, llm) @classmethod + @handle_exception async def run_text(cls, code) -> Tuple[str, str]: - try: - # We will document_store the result in this dictionary - namespace = {} - exec(code, namespace) - return namespace.get("result", ""), "" - except Exception: - # If there is an error in the code, return the error message - return "", traceback.format_exc() + # We will document_store the result in this dictionary + namespace = {} + exec(code, namespace) + return namespace.get("result", ""), "" @classmethod async def run_script(cls, working_directory, additional_python_paths=[], command=[]) -> Tuple[str, str]: @@ -145,18 +142,17 @@ class RunCode(Action): rsp = await self._aask(prompt) return RunCodeResult(summary=rsp, stdout=outs, stderr=errs) + @staticmethod + @handle_exception(exception_type=subprocess.CalledProcessError) + def _install_via_subprocess(cmd, check, cwd, env): + return subprocess.run(cmd, check=check, cwd=cwd, env=env) + @staticmethod def _install_dependencies(working_directory, env): install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] logger.info(" ".join(install_command)) - try: - subprocess.run(install_command, check=True, cwd=working_directory, env=env) - except subprocess.CalledProcessError as e: - logger.warning(f"{e}") + RunCode._install_via_subprocess(install_command, check=True, cwd=working_directory, env=env) install_pytest_command = ["python", "-m", "pip", "install", "pytest"] logger.info(" ".join(install_pytest_command)) - try: - subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) - except subprocess.CalledProcessError as e: - logger.warning(f"{e}") + RunCode._install_via_subprocess(install_pytest_command, check=True, cwd=working_directory, env=env) diff --git a/metagpt/config.py b/metagpt/config.py index 19bd02c87..45a560209 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -137,6 +137,7 @@ class Config(metaclass=Singleton): continue configs.update(yaml_data) OPTIONS.set(configs) + logger.info(f"Default OpenAI API Model: {self.openai_api_model}") @staticmethod def _get(*args, **kwargs): diff --git a/metagpt/repo_parser.py b/metagpt/repo_parser.py index b84dbab9a..9a1218ef1 100644 --- a/metagpt/repo_parser.py +++ b/metagpt/repo_parser.py @@ -15,17 +15,17 @@ from pydantic import BaseModel, Field from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.utils.exceptions import handle_exception class RepoParser(BaseModel): base_directory: Path = Field(default=None) - def parse_file(self, file_path): + @classmethod + @handle_exception(exception_type=Exception, default_return=[]) + def _parse_file(cls, file_path: Path) -> list: """Parse a Python file in the repository.""" - try: - return ast.parse(file_path.read_text()).body - except: - return [] + return ast.parse(file_path.read_text()).body def extract_class_and_function_info(self, tree, file_path): """Extract class, function, and global variable information from the AST.""" @@ -52,7 +52,7 @@ class RepoParser(BaseModel): files_classes = [] directory = self.base_directory for path in directory.rglob("*.py"): - tree = self.parse_file(path) + tree = self._parse_file(path) file_info = self.extract_class_and_function_info(tree, path) files_classes.append(file_info) @@ -90,5 +90,10 @@ def main(): logger.info(pformat(symbols)) +def error(): + """raise Exception and logs it""" + RepoParser._parse_file(Path("test.py")) + + if __name__ == "__main__": - main() + error() diff --git a/metagpt/schema.py b/metagpt/schema.py index 758149efa..7359084f5 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -21,7 +21,7 @@ import uuid from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError from pathlib import Path -from typing import Dict, List, Optional, Set, TypedDict +from typing import Dict, List, Optional, Set, Type, TypedDict, TypeVar from pydantic import BaseModel, Field @@ -36,6 +36,7 @@ from metagpt.const import ( ) from metagpt.logs import logger from metagpt.utils.common import any_to_str, any_to_str_set +from metagpt.utils.exceptions import handle_exception class RawMessage(TypedDict): @@ -160,14 +161,11 @@ class Message(BaseModel): return self.json(exclude_none=True) @staticmethod + @handle_exception(exception_type=JSONDecodeError, default_return=None) def load(val): """Convert the json string to object.""" - try: - d = json.loads(val) - return Message(**d) - except JSONDecodeError as err: - logger.error(f"parse json failed: {val}, error:{err}") - return None + d = json.loads(val) + return Message(**d) class UserMessage(Message): @@ -249,50 +247,46 @@ class MessageQueue: return json.dumps(lst) @staticmethod - def load(self, v) -> "MessageQueue": + def load(i) -> "MessageQueue": """Convert the json string to the `MessageQueue` object.""" - q = MessageQueue() + queue = MessageQueue() try: - lst = json.loads(v) + lst = json.loads(i) for i in lst: msg = Message(**i) - q.push(msg) + queue.push(msg) except JSONDecodeError as e: - logger.warning(f"JSON load failed: {v}, error:{e}") + logger.warning(f"JSON load failed: {i}, error:{e}") - return q + return queue -class CodingContext(BaseModel): +# 定义一个泛型类型变量 +T = TypeVar("T", bound="BaseModel") + + +class BaseContext(BaseModel): + @staticmethod + @handle_exception + def loads(val: str, cls: Type[T]) -> Optional[T]: + m = json.loads(val) + return cls(**m) + + +class CodingContext(BaseContext): filename: str design_doc: Optional[Document] task_doc: Optional[Document] code_doc: Optional[Document] - @staticmethod - def loads(val: str) -> CodingContext | None: - try: - m = json.loads(val) - return CodingContext(**m) - except Exception: - return None - -class TestingContext(BaseModel): +class TestingContext(BaseContext): filename: str code_doc: Document test_doc: Optional[Document] - @staticmethod - def loads(val: str) -> TestingContext | None: - try: - m = json.loads(val) - return TestingContext(**m) - except Exception: - return None - -class RunCodeContext(BaseModel): +class RunCodeContext(BaseContext): mode: str = "script" code: Optional[str] code_filename: str = "" @@ -304,28 +298,12 @@ class RunCodeContext(BaseModel): output_filename: Optional[str] output: Optional[str] - @staticmethod - def loads(val: str) -> RunCodeContext | None: - try: - m = json.loads(val) - return RunCodeContext(**m) - except Exception: - return None - -class RunCodeResult(BaseModel): +class RunCodeResult(BaseContext): summary: str stdout: str stderr: str - @staticmethod - def loads(val: str) -> RunCodeResult | None: - try: - m = json.loads(val) - return RunCodeResult(**m) - except Exception: - return None - class CodeSummarizeContext(BaseModel): design_filename: str = "" @@ -349,5 +327,5 @@ class CodeSummarizeContext(BaseModel): return hash((self.design_filename, self.task_filename)) -class BugFixContext(BaseModel): +class BugFixContext(BaseContext): filename: str = "" diff --git a/metagpt/tools/search_engine_meilisearch.py b/metagpt/tools/search_engine_meilisearch.py index f7c1c685a..ea6db4dbd 100644 --- a/metagpt/tools/search_engine_meilisearch.py +++ b/metagpt/tools/search_engine_meilisearch.py @@ -11,6 +11,8 @@ from typing import List import meilisearch from meilisearch.index import Index +from metagpt.utils.exceptions import handle_exception + class DataSource: def __init__(self, name: str, url: str): @@ -34,11 +36,7 @@ class MeilisearchEngine: index.add_documents(documents) self.set_index(index) + @handle_exception(exception_type=Exception, default_return=[]) def search(self, query): - try: - search_results = self._index.search(query) - return search_results["hits"] - except Exception as e: - # Handle MeiliSearch API errors - print(f"MeiliSearch API error: {e}") - return [] + search_results = self._index.search(query) + return search_results["hits"] diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index cdabe96a3..bf435b74f 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -20,11 +20,13 @@ import re import typing from typing import List, Tuple, Union +import aiofiles import loguru from tenacity import RetryCallState, _utils from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.logs import logger +from metagpt.utils.exceptions import handle_exception def check_cmd_exists(command) -> int: @@ -399,3 +401,11 @@ def general_after_log(i: "loguru.Logger", sec_format: str = "%0.3f") -> typing.C ) return log_it + + +@handle_exception +async def aread(file_path: str) -> str: + """Read file asynchronously.""" + async with aiofiles.open(str(file_path), mode="r") as reader: + content = await reader.read() + return content diff --git a/metagpt/utils/custom_decoder.py b/metagpt/utils/custom_decoder.py index 373d16356..eb01a1115 100644 --- a/metagpt/utils/custom_decoder.py +++ b/metagpt/utils/custom_decoder.py @@ -25,7 +25,7 @@ def py_make_scanner(context): except IndexError: raise StopIteration(idx) from None - if nextchar == '"' or nextchar == "'": + if nextchar in ("'", '"'): if idx + 2 < len(string) and string[idx + 1] == nextchar and string[idx + 2] == nextchar: # Handle the case where the next two characters are the same as nextchar return parse_string(string, idx + 3, strict, delimiter=nextchar * 3) # triple quote diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index e8347d567..d03444f0e 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -15,7 +15,8 @@ from typing import Set import aiofiles from metagpt.config import CONFIG -from metagpt.logs import logger +from metagpt.utils.common import aread +from metagpt.utils.exceptions import handle_exception class DependencyFile: @@ -36,21 +37,14 @@ class DependencyFile: """Load dependencies from the file asynchronously.""" if not self._filename.exists(): return - try: - async with aiofiles.open(str(self._filename), mode="r") as reader: - data = await reader.read() - self._dependencies = json.loads(data) - except Exception as e: - logger.error(f"Failed to load {str(self._filename)}, error:{e}") + self._dependencies = await aread(self._filename) + @handle_exception async def save(self): """Save dependencies to the file asynchronously.""" - try: - data = json.dumps(self._dependencies) - async with aiofiles.open(str(self._filename), mode="w") as writer: - await writer.write(data) - except Exception as e: - logger.error(f"Failed to save {str(self._filename)}, error:{e}") + data = json.dumps(self._dependencies) + async with aiofiles.open(str(self._filename), mode="w") as writer: + await writer.write(data) async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): """Update dependencies for a file asynchronously. diff --git a/metagpt/utils/exceptions.py b/metagpt/utils/exceptions.py new file mode 100644 index 000000000..b4b5aa590 --- /dev/null +++ b/metagpt/utils/exceptions.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/19 14:46 +@Author : alexanderwu +@File : exceptions.py +""" + + +import asyncio +import functools +import traceback +from typing import Any, Callable, Tuple, Type, TypeVar, Union + +from metagpt.logs import logger + +ReturnType = TypeVar("ReturnType") + + +def handle_exception( + _func: Callable[..., ReturnType] = None, + *, + exception_type: Union[Type[Exception], Tuple[Type[Exception], ...]] = Exception, + default_return: Any = None, +) -> Callable[..., ReturnType]: + """handle exception, return default value""" + + def decorator(func: Callable[..., ReturnType]) -> Callable[..., ReturnType]: + @functools.wraps(func) + async def async_wrapper(*args: Any, **kwargs: Any) -> ReturnType: + try: + return await func(*args, **kwargs) + except exception_type as e: + logger.opt(depth=1).error( + f"Calling {func.__name__} with args: {args}, kwargs: {kwargs} failed: {e}, " + f"stack: {traceback.format_exc()}" + ) + return default_return + + @functools.wraps(func) + def sync_wrapper(*args: Any, **kwargs: Any) -> ReturnType: + try: + return func(*args, **kwargs) + except exception_type as e: + logger.opt(depth=1).error( + f"Calling {func.__name__} with args: {args}, kwargs: {kwargs} failed: {e}, " + f"stack: {traceback.format_exc()}" + ) + return default_return + + if asyncio.iscoroutinefunction(func): + return async_wrapper + else: + return sync_wrapper + + if _func is None: + return decorator + else: + return decorator(_func) diff --git a/metagpt/utils/file.py b/metagpt/utils/file.py index 6bb9a1a97..f62b44eb8 100644 --- a/metagpt/utils/file.py +++ b/metagpt/utils/file.py @@ -11,6 +11,7 @@ from pathlib import Path import aiofiles from metagpt.logs import logger +from metagpt.utils.exceptions import handle_exception class File: @@ -19,6 +20,7 @@ class File: CHUNK_SIZE = 64 * 1024 @classmethod + @handle_exception async def write(cls, root_path: Path, filename: str, content: bytes) -> Path: """Write the file content to the local specified path. @@ -33,18 +35,15 @@ class File: Raises: Exception: If an unexpected error occurs during the file writing process. """ - try: - root_path.mkdir(parents=True, exist_ok=True) - full_path = root_path / filename - async with aiofiles.open(full_path, mode="wb") as writer: - await writer.write(content) - logger.debug(f"Successfully write file: {full_path}") - return full_path - except Exception as e: - logger.error(f"Error writing file: {e}") - raise e + root_path.mkdir(parents=True, exist_ok=True) + full_path = root_path / filename + async with aiofiles.open(full_path, mode="wb") as writer: + await writer.write(content) + logger.debug(f"Successfully write file: {full_path}") + return full_path @classmethod + @handle_exception async def read(cls, file_path: Path, chunk_size: int = None) -> bytes: """Partitioning read the file content from the local specified path. @@ -58,18 +57,14 @@ class File: Raises: Exception: If an unexpected error occurs during the file reading process. """ - try: - chunk_size = chunk_size or cls.CHUNK_SIZE - async with aiofiles.open(file_path, mode="rb") as reader: - chunks = list() - while True: - chunk = await reader.read(chunk_size) - if not chunk: - break - chunks.append(chunk) - content = b"".join(chunks) - logger.debug(f"Successfully read file, the path of file: {file_path}") - return content - except Exception as e: - logger.error(f"Error reading file: {e}") - raise e + chunk_size = chunk_size or cls.CHUNK_SIZE + async with aiofiles.open(file_path, mode="rb") as reader: + chunks = list() + while True: + chunk = await reader.read(chunk_size) + if not chunk: + break + chunks.append(chunk) + content = b"".join(chunks) + logger.debug(f"Successfully read file, the path of file: {file_path}") + return content diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 2eca799a8..099556a6b 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -19,6 +19,7 @@ import aiofiles from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Document +from metagpt.utils.common import aread from metagpt.utils.json_to_markdown import json_to_markdown @@ -97,15 +98,7 @@ class FileRepository: path_name = self.workdir / filename if not path_name.exists(): return None - try: - async with aiofiles.open(str(path_name), mode="r") as reader: - doc.content = await reader.read() - except FileNotFoundError as e: - logger.info(f"open {str(path_name)} failed:{e}") - return None - except Exception as e: - logger.info(f"open {str(path_name)} failed:{e}") - return None + doc.content = await aread(path_name) return doc async def get_all(self) -> List[Document]: From d5d7db0bf80d6210bbf71d5f274fe49c5ef7575d Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 16:22:29 +0800 Subject: [PATCH 15/49] bug fix and proper log --- metagpt/config.py | 4 ++-- metagpt/utils/dependency_file.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 45a560209..629a5b797 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -48,6 +48,7 @@ class Config(metaclass=Singleton): self._init_with_config_files_and_env(yaml_file) logger.debug("Config loading done.") self._update() + logger.info(f"OpenAI API Model: {self.openai_api_model}") def _update(self): # logger.info("Config loading done.") @@ -74,7 +75,7 @@ class Config(metaclass=Singleton): self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) - self.openai_api_model = self._get("OPENAI_API_MODEL", "gpt-4") + self.openai_api_model = self._get("OPENAI_API_MODEL", "gpt-4-1106-preview") self.max_tokens_rsp = self._get("MAX_TOKENS", 2048) self.deployment_name = self._get("DEPLOYMENT_NAME") self.deployment_id = self._get("DEPLOYMENT_ID") @@ -137,7 +138,6 @@ class Config(metaclass=Singleton): continue configs.update(yaml_data) OPTIONS.set(configs) - logger.info(f"Default OpenAI API Model: {self.openai_api_model}") @staticmethod def _get(*args, **kwargs): diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py index d03444f0e..8a6575e9e 100644 --- a/metagpt/utils/dependency_file.py +++ b/metagpt/utils/dependency_file.py @@ -37,7 +37,7 @@ class DependencyFile: """Load dependencies from the file asynchronously.""" if not self._filename.exists(): return - self._dependencies = await aread(self._filename) + self._dependencies = json.loads(await aread(self._filename)) @handle_exception async def save(self): From 2c1538f35035a5732137ef760a1b86d2bac50ada Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 16:31:38 +0800 Subject: [PATCH 16/49] bug fix and proper log --- metagpt/schema.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 7359084f5..b24f114b0 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -266,11 +266,11 @@ T = TypeVar("T", bound="BaseModel") class BaseContext(BaseModel): - @staticmethod + @classmethod @handle_exception - def loads(val: str, cls: Type[T]) -> Optional[T]: - m = json.loads(val) - return cls(**m) + def loads(cls: Type[T], val: str) -> Optional[T]: + i = json.loads(val) + return cls(**i) class CodingContext(BaseContext): From e67dbc92e42fcc9479027df6e9d00eabce20df36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 19 Dec 2023 16:37:01 +0800 Subject: [PATCH 17/49] feat: disable -- max_auto_summarize_code --- metagpt/startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/startup.py b/metagpt/startup.py index f930c386b..e886ad2a4 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -26,7 +26,7 @@ def startup( ), reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), max_auto_summarize_code: int = typer.Option( - default=-1, + default=0, help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the workflow.", ), ): From 93745b85ccfbe7b953c17a36867dc823ff2699c5 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 16:54:06 +0800 Subject: [PATCH 18/49] refine config --- config/config.yaml | 2 +- metagpt/config.py | 51 +++++++++++++++++++------------ metagpt/provider/anthropic_api.py | 4 +-- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 8fd208c59..9a7207c1a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -20,7 +20,7 @@ RPM: 10 #SPARK_URL : "ws://spark-api.xf-yun.com/v2.1/chat" #### if Anthropic -#Anthropic_API_KEY: "YOUR_API_KEY" +#ANTHROPIC_API_KEY: "YOUR_API_KEY" #### if AZURE, check https://github.com/openai/openai-cookbook/blob/main/examples/azure/chat.ipynb #### You can use ENGINE or DEPLOYMENT mode diff --git a/metagpt/config.py b/metagpt/config.py index 629a5b797..702a2ddc9 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -46,30 +46,41 @@ class Config(metaclass=Singleton): def __init__(self, yaml_file=default_yaml_file): self._init_with_config_files_and_env(yaml_file) - logger.debug("Config loading done.") self._update() + logger.debug("Config loading done.") logger.info(f"OpenAI API Model: {self.openai_api_model}") + @staticmethod + def _is_valid_llm_key(k) -> bool: + return k and k != "YOUR_API_KEY" + + def _check_llm_exists(self): + if not any( + [ + self._is_valid_llm_key(self.openai_api_key), + self._is_valid_llm_key(self.anthropic_api_key), + self._is_valid_llm_key(self.zhipuai_api_key), + self._is_valid_llm_key(self.fireworks_api_key), + self.open_llm_api_base, + ] + ): + raise NotConfiguredException( + "Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY " + "or FIREWORKS_API_KEY or OPEN_LLM_API_BASE" + ) + def _update(self): # logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") + self.openai_api_key = self._get("OPENAI_API_KEY") - self.anthropic_api_key = self._get("Anthropic_API_KEY") + self.anthropic_api_key = self._get("ANTHROPIC_API_KEY") self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY") self.open_llm_api_base = self._get("OPEN_LLM_API_BASE") self.open_llm_api_model = self._get("OPEN_LLM_API_MODEL") self.fireworks_api_key = self._get("FIREWORKS_API_KEY") - if ( - (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) - and (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key) - and (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key) - and (not self.open_llm_api_base) - and (not self.fireworks_api_key or "YOUR_API_KEY" == self.fireworks_api_key) - ): - raise NotConfiguredException( - "Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first " - "or FIREWORKS_API_KEY or OPEN_LLM_API_BASE" - ) + self._check_llm_exists() + self.openai_api_base = self._get("OPENAI_API_BASE") self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy self.openai_api_type = self._get("OPENAI_API_TYPE") @@ -89,7 +100,7 @@ class Config(metaclass=Singleton): self.fireworks_api_base = self._get("FIREWORKS_API_BASE") self.fireworks_api_model = self._get("FIREWORKS_API_MODEL") - self.claude_api_key = self._get("Anthropic_API_KEY") + self.claude_api_key = self._get("ANTHROPIC_API_KEY") self.serpapi_api_key = self._get("SERPAPI_API_KEY") self.serper_api_key = self._get("SERPER_API_KEY") self.google_api_key = self._get("GOOGLE_API_KEY") @@ -141,8 +152,8 @@ class Config(metaclass=Singleton): @staticmethod def _get(*args, **kwargs): - m = OPTIONS.get() - return m.get(*args, **kwargs) + i = OPTIONS.get() + return i.get(*args, **kwargs) def get(self, key, *args, **kwargs): """Search for a value in config/key.yaml, config/config.yaml, and env; raise an error if not found""" @@ -155,8 +166,8 @@ class Config(metaclass=Singleton): OPTIONS.get()[name] = value def __getattr__(self, name: str) -> Any: - m = OPTIONS.get() - return m.get(name) + i = OPTIONS.get() + return i.get(name) def set_context(self, options: dict): """Update current config""" @@ -175,8 +186,8 @@ class Config(metaclass=Singleton): def new_environ(self): """Return a new os.environ object""" env = os.environ.copy() - m = self.options - env.update({k: v for k, v in m.items() if isinstance(v, str)}) + i = self.options + env.update({k: v for k, v in i.items() if isinstance(v, str)}) return env diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 03802a716..f5b06c855 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -14,7 +14,7 @@ from metagpt.config import CONFIG class Claude2: def ask(self, prompt): - client = Anthropic(api_key=CONFIG.claude_api_key) + client = Anthropic(api_key=CONFIG.anthropic_api_key) res = client.completions.create( model="claude-2", @@ -24,7 +24,7 @@ class Claude2: return res.completion async def aask(self, prompt): - client = Anthropic(api_key=CONFIG.claude_api_key) + client = Anthropic(api_key=CONFIG.anthropic_api_key) res = client.completions.create( model="claude-2", From 7f04ec2060da2ccdc3ca72a4d5e7e60377958b7d Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 17:06:07 +0800 Subject: [PATCH 19/49] refine code --- metagpt/config.py | 8 ++++++++ metagpt/repo_parser.py | 2 +- metagpt/startup.py | 9 +++------ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 702a2ddc9..48ac82a3a 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -130,6 +130,14 @@ class Config(metaclass=Singleton): self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) self._ensure_workspace_exists() + def update_via_cli(self, project_path, project_name, inc, reqa_file, max_auto_summarize_code): + """update config via cli""" + self.project_path = project_path + self.project_name = project_name + self.inc = inc + self.reqa_file = reqa_file + self.max_auto_summarize_code = max_auto_summarize_code + def _ensure_workspace_exists(self): self.workspace_path.mkdir(parents=True, exist_ok=True) logger.debug(f"WORKSPACE_PATH set to {self.workspace_path}") diff --git a/metagpt/repo_parser.py b/metagpt/repo_parser.py index 9a1218ef1..3524a5bce 100644 --- a/metagpt/repo_parser.py +++ b/metagpt/repo_parser.py @@ -96,4 +96,4 @@ def error(): if __name__ == "__main__": - error() + main() diff --git a/metagpt/startup.py b/metagpt/startup.py index f930c386b..047f35cf6 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -27,7 +27,8 @@ def startup( reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), max_auto_summarize_code: int = typer.Option( default=-1, - help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the workflow.", + help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating " + "unlimited. This parameter is used for debugging the workflow.", ), ): """Run a startup. Be a boss.""" @@ -41,14 +42,10 @@ def startup( from metagpt.team import Team # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135. - CONFIG.project_path = project_path if project_path: inc = True project_name = project_name or Path(project_path).name - CONFIG.project_name = project_name - CONFIG.inc = inc - CONFIG.reqa_file = reqa_file - CONFIG.max_auto_summarize_code = max_auto_summarize_code + CONFIG.update_via_cli(project_path, project_name, inc, reqa_file, max_auto_summarize_code) company = Team() company.hire( From 2bae7f2bfb116d9deeab3e6d6237da0a12bdd2be Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 17:11:02 +0800 Subject: [PATCH 20/49] refine code --- metagpt/config.py | 13 +++++++++++++ metagpt/startup.py | 5 ----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 48ac82a3a..bdf580a1f 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -45,6 +45,7 @@ class Config(metaclass=Singleton): default_yaml_file = METAGPT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): + self._init_cli_paras() self._init_with_config_files_and_env(yaml_file) self._update() logger.debug("Config loading done.") @@ -130,8 +131,20 @@ class Config(metaclass=Singleton): self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) self._ensure_workspace_exists() + def _init_cli_paras(self): + self.project_path = None + self.project_name = None + self.inc = None + self.reqa_file = None + self.max_auto_summarize_code = None + def update_via_cli(self, project_path, project_name, inc, reqa_file, max_auto_summarize_code): """update config via cli""" + + # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135. + if project_path: + inc = True + project_name = project_name or Path(project_path).name self.project_path = project_path self.project_name = project_name self.inc = inc diff --git a/metagpt/startup.py b/metagpt/startup.py index 047f35cf6..37526dbcc 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import asyncio -from pathlib import Path import typer @@ -41,10 +40,6 @@ def startup( ) from metagpt.team import Team - # Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135. - if project_path: - inc = True - project_name = project_name or Path(project_path).name CONFIG.update_via_cli(project_path, project_name, inc, reqa_file, max_auto_summarize_code) company = Team() From 1213c5f88fe2ab257681d7f383e311c6bcbff925 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 17:14:50 +0800 Subject: [PATCH 21/49] fix comment --- metagpt/team.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/team.py b/metagpt/team.py index a5c405f80..ddd145269 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -21,8 +21,8 @@ from metagpt.utils.common import NoMoneyException class Team(BaseModel): """ - Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a platform for instant messaging, - dedicated to perform any multi-agent activity, such as collaboratively writing executable code. + Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a env for instant messaging, + dedicated to env any multi-agent activity, such as collaboratively writing executable code. """ env: Environment = Field(default_factory=Environment) From f27461f7582ec1143f43718ae79373187e0c7684 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 17:55:34 +0800 Subject: [PATCH 22/49] add llm provider registry --- metagpt/config.py | 57 +++++++++++++---------- metagpt/llm.py | 21 +-------- metagpt/provider/fireworks_api.py | 4 +- metagpt/provider/llm_provider_registry.py | 34 ++++++++++++++ metagpt/provider/open_llm_api.py | 4 +- metagpt/provider/openai_api.py | 4 +- metagpt/provider/spark_api.py | 4 +- metagpt/provider/zhipuai_api.py | 4 +- metagpt/schema.py | 10 ++-- 9 files changed, 89 insertions(+), 53 deletions(-) create mode 100644 metagpt/provider/llm_provider_registry.py diff --git a/metagpt/config.py b/metagpt/config.py index bdf580a1f..a0d61b39f 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -8,6 +8,7 @@ Provide configuration, singleton """ import os from copy import deepcopy +from enum import Enum from pathlib import Path from typing import Any @@ -31,6 +32,15 @@ class NotConfiguredException(Exception): super().__init__(self.message) +class LLMProviderEnum(Enum): + OPENAI = "openai" + ANTHROPIC = "anthropic" + SPARK = "spark" + ZHIPUAI = "zhipuai" + FIREWORKS = "fireworks" + OPEN_LLM = "open_llm" + + class Config(metaclass=Singleton): """ Regular usage method: @@ -45,31 +55,37 @@ class Config(metaclass=Singleton): default_yaml_file = METAGPT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): - self._init_cli_paras() + # cli paras + self.project_path = "" + self.project_name = "" + self.inc = False + self.reqa_file = "" + self.max_auto_summarize_code = 0 + self._init_with_config_files_and_env(yaml_file) self._update() logger.debug("Config loading done.") logger.info(f"OpenAI API Model: {self.openai_api_model}") + def get_default_llm_provider_enum(self): + if self._is_valid_llm_key(self.openai_api_key): + llm = LLMProviderEnum.OPENAI + elif self._is_valid_llm_key(self.anthropic_api_key): + llm = LLMProviderEnum.ANTHROPIC + elif self._is_valid_llm_key(self.zhipuai_api_key): + llm = LLMProviderEnum.ZHIPUAI + elif self._is_valid_llm_key(self.fireworks_api_key): + llm = LLMProviderEnum.FIREWORKS + elif self.open_llm_api_base: + llm = LLMProviderEnum.OPEN_LLM + else: + raise NotConfiguredException("You should config a LLM configuration first") + return llm + @staticmethod def _is_valid_llm_key(k) -> bool: return k and k != "YOUR_API_KEY" - def _check_llm_exists(self): - if not any( - [ - self._is_valid_llm_key(self.openai_api_key), - self._is_valid_llm_key(self.anthropic_api_key), - self._is_valid_llm_key(self.zhipuai_api_key), - self._is_valid_llm_key(self.fireworks_api_key), - self.open_llm_api_base, - ] - ): - raise NotConfiguredException( - "Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY " - "or FIREWORKS_API_KEY or OPEN_LLM_API_BASE" - ) - def _update(self): # logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") @@ -80,7 +96,7 @@ class Config(metaclass=Singleton): self.open_llm_api_base = self._get("OPEN_LLM_API_BASE") self.open_llm_api_model = self._get("OPEN_LLM_API_MODEL") self.fireworks_api_key = self._get("FIREWORKS_API_KEY") - self._check_llm_exists() + _ = self.get_default_llm_provider_enum() self.openai_api_base = self._get("OPENAI_API_BASE") self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy @@ -131,13 +147,6 @@ class Config(metaclass=Singleton): self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) self._ensure_workspace_exists() - def _init_cli_paras(self): - self.project_path = None - self.project_name = None - self.inc = None - self.reqa_file = None - self.max_auto_summarize_code = None - def update_via_cli(self, project_path, project_name, inc, reqa_file, max_auto_summarize_code): """update config via cli""" diff --git a/metagpt/llm.py b/metagpt/llm.py index 7c0ad7975..e0c0716de 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -8,12 +8,8 @@ from metagpt.config import CONFIG from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.provider.fireworks_api import FireWorksGPTAPI from metagpt.provider.human_provider import HumanProvider -from metagpt.provider.open_llm_api import OpenLLMGPTAPI -from metagpt.provider.openai_api import OpenAIGPTAPI -from metagpt.provider.spark_api import SparkAPI -from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI +from metagpt.provider.llm_provider_registry import LLMProviderRegistry _ = HumanProvider() # Avoid pre-commit error @@ -21,17 +17,4 @@ _ = HumanProvider() # Avoid pre-commit error def LLM() -> BaseGPTAPI: """initialize different LLM instance according to the key field existence""" # TODO a little trick, can use registry to initialize LLM instance further - if CONFIG.openai_api_key: - llm = OpenAIGPTAPI() - elif CONFIG.spark_api_key: - llm = SparkAPI() - elif CONFIG.zhipuai_api_key: - llm = ZhiPuAIGPTAPI() - elif CONFIG.open_llm_api_base: - llm = OpenLLMGPTAPI() - elif CONFIG.fireworks_api_key: - llm = FireWorksGPTAPI() - else: - raise RuntimeError("You should config a LLM configuration first") - - return llm + return LLMProviderRegistry.get_provider(CONFIG.get_default_llm_provider_enum()) diff --git a/metagpt/provider/fireworks_api.py b/metagpt/provider/fireworks_api.py index 47ac9cf61..a76151666 100644 --- a/metagpt/provider/fireworks_api.py +++ b/metagpt/provider/fireworks_api.py @@ -4,10 +4,12 @@ import openai -from metagpt.config import CONFIG +from metagpt.config import CONFIG, LLMProviderEnum +from metagpt.provider.llm_provider_registry import register_provider from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter +@register_provider(LLMProviderEnum.FIREWORKS) class FireWorksGPTAPI(OpenAIGPTAPI): def __init__(self): self.__init_fireworks(CONFIG) diff --git a/metagpt/provider/llm_provider_registry.py b/metagpt/provider/llm_provider_registry.py new file mode 100644 index 000000000..2b3ef93a3 --- /dev/null +++ b/metagpt/provider/llm_provider_registry.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/12/19 17:26 +@Author : alexanderwu +@File : llm_provider_registry.py +""" +from metagpt.config import LLMProviderEnum + + +class LLMProviderRegistry: + def __init__(self): + self.providers = {} + + def register(self, key, provider_cls): + self.providers[key] = provider_cls + + def get_provider(self, enum: LLMProviderEnum): + """get provider instance according to the enum""" + return self.providers[enum]() + + +# Registry instance +LLM_REGISTRY = LLMProviderRegistry() + + +def register_provider(key): + """register provider to registry""" + + def decorator(cls): + LLM_REGISTRY.register(key, cls) + return cls + + return decorator diff --git a/metagpt/provider/open_llm_api.py b/metagpt/provider/open_llm_api.py index f421e30c8..bada0e294 100644 --- a/metagpt/provider/open_llm_api.py +++ b/metagpt/provider/open_llm_api.py @@ -4,8 +4,9 @@ import openai -from metagpt.config import CONFIG +from metagpt.config import CONFIG, LLMProviderEnum from metagpt.logs import logger +from metagpt.provider.llm_provider_registry import register_provider from metagpt.provider.openai_api import CostManager, OpenAIGPTAPI, RateLimiter @@ -31,6 +32,7 @@ class OpenLLMCostManager(CostManager): CONFIG.total_cost = self.total_cost +@register_provider(LLMProviderEnum.OPEN_LLM) class OpenLLMGPTAPI(OpenAIGPTAPI): def __init__(self): self.__init_openllm(CONFIG) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 86054881e..0be70b3ca 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -18,10 +18,11 @@ from tenacity import ( wait_random_exponential, ) -from metagpt.config import CONFIG +from metagpt.config import CONFIG, LLMProviderEnum from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE +from metagpt.provider.llm_provider_registry import register_provider from metagpt.schema import Message from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( @@ -137,6 +138,7 @@ See FAQ 5.8 raise retry_state.outcome.exception() +@register_provider(LLMProviderEnum.OPENAI) class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ Check https://platform.openai.com/examples for examples diff --git a/metagpt/provider/spark_api.py b/metagpt/provider/spark_api.py index 60c86f4dc..484fa7956 100644 --- a/metagpt/provider/spark_api.py +++ b/metagpt/provider/spark_api.py @@ -19,11 +19,13 @@ from wsgiref.handlers import format_date_time import websocket # 使用websocket_client -from metagpt.config import CONFIG +from metagpt.config import CONFIG, LLMProviderEnum from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.provider.llm_provider_registry import register_provider +@register_provider(LLMProviderEnum.SPARK) class SparkAPI(BaseGPTAPI): def __init__(self): logger.warning("当前方法无法支持异步运行。当你使用acompletion时，并不能并行访问。") diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 92119b764..eef0e51e1 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -16,9 +16,10 @@ from tenacity import ( wait_random_exponential, ) -from metagpt.config import CONFIG +from metagpt.config import CONFIG, LLMProviderEnum from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.provider.llm_provider_registry import register_provider from metagpt.provider.openai_api import CostManager, log_and_reraise from metagpt.provider.zhipuai.zhipu_model_api import ZhiPuModelAPI @@ -30,6 +31,7 @@ class ZhiPuEvent(Enum): FINISH = "finish" +@register_provider(LLMProviderEnum.ZHIPUAI) class ZhiPuAIGPTAPI(BaseGPTAPI): """ Refs to `https://open.bigmodel.cn/dev/api#chatglm_turbo` diff --git a/metagpt/schema.py b/metagpt/schema.py index b24f114b0..aacc2cebb 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -164,8 +164,8 @@ class Message(BaseModel): @handle_exception(exception_type=JSONDecodeError, default_return=None) def load(val): """Convert the json string to object.""" - d = json.loads(val) - return Message(**d) + i = json.loads(val) + return Message(**i) class UserMessage(Message): @@ -247,16 +247,16 @@ class MessageQueue: return json.dumps(lst) @staticmethod - def load(i) -> "MessageQueue": + def load(data) -> "MessageQueue": """Convert the json string to the `MessageQueue` object.""" queue = MessageQueue() try: - lst = json.loads(i) + lst = json.loads(data) for i in lst: msg = Message(**i) queue.push(msg) except JSONDecodeError as e: - logger.warning(f"JSON load failed: {i}, error:{e}") + logger.warning(f"JSON load failed: {data}, error:{e}") return queue From 25b8a6dcef768ed1e45489e2dd3a5462f37fd593 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 18:02:51 +0800 Subject: [PATCH 23/49] make registry work --- metagpt/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index e0c0716de..60f110a00 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -9,7 +9,7 @@ from metagpt.config import CONFIG from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.human_provider import HumanProvider -from metagpt.provider.llm_provider_registry import LLMProviderRegistry +from metagpt.provider.llm_provider_registry import LLM_REGISTRY _ = HumanProvider() # Avoid pre-commit error @@ -17,4 +17,4 @@ _ = HumanProvider() # Avoid pre-commit error def LLM() -> BaseGPTAPI: """initialize different LLM instance according to the key field existence""" # TODO a little trick, can use registry to initialize LLM instance further - return LLMProviderRegistry.get_provider(CONFIG.get_default_llm_provider_enum()) + return LLM_REGISTRY.get_provider(CONFIG.get_default_llm_provider_enum()) From 77735d6e612422911dedd86c40aebb2b7c69dcb3 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 18:04:12 +0800 Subject: [PATCH 24/49] make registry work --- metagpt/llm.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 60f110a00..8763642f0 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -6,7 +6,7 @@ @File : llm.py """ -from metagpt.config import CONFIG +from metagpt.config import CONFIG, LLMProviderEnum from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.human_provider import HumanProvider from metagpt.provider.llm_provider_registry import LLM_REGISTRY @@ -14,7 +14,6 @@ from metagpt.provider.llm_provider_registry import LLM_REGISTRY _ = HumanProvider() # Avoid pre-commit error -def LLM() -> BaseGPTAPI: - """initialize different LLM instance according to the key field existence""" - # TODO a little trick, can use registry to initialize LLM instance further - return LLM_REGISTRY.get_provider(CONFIG.get_default_llm_provider_enum()) +def LLM(provider: LLMProviderEnum = CONFIG.get_default_llm_provider_enum()) -> BaseGPTAPI: + """get the default llm provider""" + return LLM_REGISTRY.get_provider(provider) From 3baf47a3d64ebf9278ec5bee5e6ec524fdf9f666 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 18:50:55 +0800 Subject: [PATCH 25/49] refine code for isinstance --- metagpt/actions/write_prd.py | 2 +- metagpt/roles/role.py | 2 +- metagpt/roles/searcher.py | 2 +- metagpt/utils/common.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index bb0cf8fb9..adba7decb 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -182,7 +182,7 @@ class WritePRD(Action): return if not CONFIG.project_name: - if isinstance(prd, ActionOutput) or isinstance(prd, ActionNode): + if isinstance(prd, (ActionOutput, ActionNode)): ws_name = prd.instruct_content.dict()["Project Name"] else: ws_name = CodeParser.parse_str(block="Project Name", text=prd) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 48688ad5f..e13bf454b 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -267,7 +267,7 @@ class Role: async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.important_memory) - if isinstance(response, ActionOutput) or isinstance(response, ActionNode): + if isinstance(response, (ActionOutput, ActionNode)): msg = Message( content=response.content, instruct_content=response.instruct_content, diff --git a/metagpt/roles/searcher.py b/metagpt/roles/searcher.py index 5760202ff..31de8e896 100644 --- a/metagpt/roles/searcher.py +++ b/metagpt/roles/searcher.py @@ -59,7 +59,7 @@ class Searcher(Role): logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.memory.get(k=0)) - if isinstance(response, ActionOutput) or isinstance(response, ActionNode): + if isinstance(response, (ActionOutput, ActionNode)): msg = Message( content=response.content, instruct_content=response.instruct_content, diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index bf435b74f..fa18694e3 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -197,7 +197,7 @@ class OutputParser: result = ast.literal_eval(structure_text) # Ensure the result matches the specified data type - if isinstance(result, list) or isinstance(result, dict): + if isinstance(result, (list, dict)): return result raise ValueError(f"The extracted structure is not a {data_type}.") From 5aa4ef5d836771b3335ded771626e44dfce74c2c Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 18:54:04 +0800 Subject: [PATCH 26/49] fix typo --- metagpt/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index d4e85ca7b..766024222 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -55,8 +55,7 @@ class Config(metaclass=Singleton): default_yaml_file = METAGPT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): - - golbal_options = OPTIONS.get() + global_options = OPTIONS.get() # cli paras self.project_path = "" self.project_name = "" @@ -66,7 +65,7 @@ class Config(metaclass=Singleton): self._init_with_config_files_and_env(yaml_file) self._update() - golbal_options.update(OPTIONS.get()) + global_options.update(OPTIONS.get()) logger.debug("Config loading done.") logger.info(f"OpenAI API Model: {self.openai_api_model}") From 9d1b628bce1de85b401bbb781c75707f7774dfba Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 19:00:20 +0800 Subject: [PATCH 27/49] refine cli --- metagpt/startup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/startup.py b/metagpt/startup.py index a89b9c5e9..d6f3397bc 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -6,7 +6,7 @@ import typer from metagpt.config import CONFIG -app = typer.Typer() +app = typer.Typer(add_completion=False) @app.command() @@ -23,7 +23,9 @@ def startup( default="", help="Specify the directory path of the old version project to fulfill the " "incremental requirements.", ), - reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."), + reqa_file: str = typer.Option( + default="", help="Specify the source file name for rewriting the quality assurance " "code." + ), max_auto_summarize_code: int = typer.Option( default=0, help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating " From 505133cacc587c5894f10bed149d774c41b857e2 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 19:00:39 +0800 Subject: [PATCH 28/49] refine cli --- metagpt/startup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/startup.py b/metagpt/startup.py index d6f3397bc..a1af90ffc 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -21,10 +21,10 @@ def startup( inc: bool = typer.Option(default=False, help="Incremental mode. Use it to coop with existing repo."), project_path: str = typer.Option( default="", - help="Specify the directory path of the old version project to fulfill the " "incremental requirements.", + help="Specify the directory path of the old version project to fulfill the incremental requirements.", ), reqa_file: str = typer.Option( - default="", help="Specify the source file name for rewriting the quality assurance " "code." + default="", help="Specify the source file name for rewriting the quality assurance code." ), max_auto_summarize_code: int = typer.Option( default=0, From 6dfa4e2c9e44d8db8e8e1c67646ae88d4547c968 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 19:15:30 +0800 Subject: [PATCH 29/49] fix pylint --- examples/agent_creator.py | 9 ++++----- metagpt/memory/longterm_memory.py | 10 +++++----- metagpt/memory/memory_storage.py | 2 +- metagpt/roles/product_manager.py | 2 +- metagpt/roles/qa_engineer.py | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/examples/agent_creator.py b/examples/agent_creator.py index 05417d24a..26af8a287 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -12,9 +12,8 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -with open(METAGPT_ROOT / "examples/build_customized_agent.py", "r") as f: - # use official example script to guide AgentCreator - MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read() +EXAMPLE_CODE_FILE = METAGPT_ROOT / "examples/build_customized_agent.py" +MULTI_ACTION_AGENT_CODE_EXAMPLE = EXAMPLE_CODE_FILE.read_text() class CreateAgent(Action): @@ -50,8 +49,8 @@ class CreateAgent(Action): match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else "" CONFIG.workspace_path.mkdir(parents=True, exist_ok=True) - with open(CONFIG.workspace_path / "agent_created_agent.py", "w") as f: - f.write(code_text) + new_file = CONFIG.workspace_path / "agent_created_agent.py" + new_file.write_text(code_text) return code_text diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 22032a86e..ab2214261 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -19,7 +19,7 @@ class LongTermMemory(Memory): def __init__(self): self.memory_storage: MemoryStorage = MemoryStorage() - super(LongTermMemory, self).__init__() + super().__init__() self.rc = None # RoleContext self.msg_from_recover = False @@ -37,7 +37,7 @@ class LongTermMemory(Memory): self.msg_from_recover = False def add(self, message: Message): - super(LongTermMemory, self).add(message) + super().add(message) for action in self.rc.watch: if message.cause_by == action and not self.msg_from_recover: # currently, only add role's watching messages to its memory_storage @@ -50,7 +50,7 @@ class LongTermMemory(Memory): 1. find the short-term memory(stm) news 2. furthermore, filter out similar messages based on ltm(long-term memory), get the final news """ - stm_news = super(LongTermMemory, self).find_news(observed, k=k) # shot-term memory news + stm_news = super().find_news(observed, k=k) # shot-term memory news if not self.memory_storage.is_initialized: # memory_storage hasn't initialized, use default `find_news` to get stm_news return stm_news @@ -64,9 +64,9 @@ class LongTermMemory(Memory): return ltm_news[-k:] def delete(self, message: Message): - super(LongTermMemory, self).delete(message) + super().delete(message) # TODO delete message in memory_storage def clear(self): - super(LongTermMemory, self).clear() + super().clear() self.memory_storage.clean() diff --git a/metagpt/memory/memory_storage.py b/metagpt/memory/memory_storage.py index a213f6d7a..fafb33568 100644 --- a/metagpt/memory/memory_storage.py +++ b/metagpt/memory/memory_storage.py @@ -58,7 +58,7 @@ class MemoryStorage(FaissStore): return index_fpath, storage_fpath def persist(self): - super(MemoryStorage, self).persist() + super().persist() logger.debug(f"Agent {self.role_id} persist memory into local") def add(self, message: Message) -> bool: diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index e5e9f2b5e..7858d2caa 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -54,4 +54,4 @@ class ProductManager(Role): return self._rc.todo async def _observe(self, ignore_memory=False) -> int: - return await super(ProductManager, self)._observe(ignore_memory=True) + return await super()._observe(ignore_memory=True) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 4439b9b19..71b474a3b 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -178,4 +178,4 @@ class QaEngineer(Role): async def _observe(self, ignore_memory=False) -> int: # This role has events that trigger and execute themselves based on conditions, and cannot rely on the # content of memory to activate. - return await super(QaEngineer, self)._observe(ignore_memory=True) + return await super()._observe(ignore_memory=True) From c12cd7b9c6bd2d900fbd70072cd9731b86486e1b Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 19:25:01 +0800 Subject: [PATCH 30/49] refine code --- metagpt/config.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 766024222..80a3a28f4 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -67,25 +67,23 @@ class Config(metaclass=Singleton): self._update() global_options.update(OPTIONS.get()) logger.debug("Config loading done.") - logger.info(f"OpenAI API Model: {self.openai_api_model}") - def get_default_llm_provider_enum(self): - if self._is_valid_llm_key(self.openai_api_key): - llm = LLMProviderEnum.OPENAI - elif self._is_valid_llm_key(self.anthropic_api_key): - llm = LLMProviderEnum.ANTHROPIC - elif self._is_valid_llm_key(self.zhipuai_api_key): - llm = LLMProviderEnum.ZHIPUAI - elif self._is_valid_llm_key(self.fireworks_api_key): - llm = LLMProviderEnum.FIREWORKS - elif self.open_llm_api_base: - llm = LLMProviderEnum.OPEN_LLM - else: - raise NotConfiguredException("You should config a LLM configuration first") - return llm + def get_default_llm_provider_enum(self) -> LLMProviderEnum: + for k, v in [ + (self.openai_api_key, LLMProviderEnum.OPENAI), + (self.anthropic_api_key, LLMProviderEnum.ANTHROPIC), + (self.zhipuai_api_key, LLMProviderEnum.ZHIPUAI), + (self.fireworks_api_key, LLMProviderEnum.FIREWORKS), + (self.open_llm_api_base, LLMProviderEnum.OPEN_LLM), # reuse logic. but not a key + ]: + if self._is_valid_llm_key(k): + if self.openai_api_model: + logger.info(f"OpenAI API Model: {self.openai_api_model}") + return v + raise NotConfiguredException("You should config a LLM configuration first") @staticmethod - def _is_valid_llm_key(k) -> bool: + def _is_valid_llm_key(k: str) -> bool: return k and k != "YOUR_API_KEY" def _update(self): From edb90690263b5b0aa91ecdf61e94476e6ff613c4 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 19:26:01 +0800 Subject: [PATCH 31/49] delete manager.py --- metagpt/manager.py | 66 ---------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 metagpt/manager.py diff --git a/metagpt/manager.py b/metagpt/manager.py deleted file mode 100644 index a063608be..000000000 --- a/metagpt/manager.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/11 14:42 -@Author : alexanderwu -@File : manager.py -""" -from metagpt.llm import LLM -from metagpt.logs import logger -from metagpt.schema import Message - - -class Manager: - def __init__(self, llm: LLM = LLM()): - self.llm = llm # Large Language Model - self.role_directions = { - "User": "Product Manager", - "Product Manager": "Architect", - "Architect": "Engineer", - "Engineer": "QA Engineer", - "QA Engineer": "Product Manager", - } - self.prompt_template = """ - Given the following message: - {message} - - And the current status of roles: - {roles} - - Which role should handle this message? - """ - - async def handle(self, message: Message, environment): - """ - 管理员处理信息，现在简单的将信息递交给下一个人 - The administrator processes the information, now simply passes the information on to the next person - :param message: - :param environment: - :return: - """ - # Get all roles from the environment - roles = environment.get_roles() - # logger.debug(f"{roles=}, {message=}") - - # Build a context for the LLM to understand the situation - # context = { - # "message": str(message), - # "roles": {role.name: role.get_info() for role in roles}, - # } - # Ask the LLM to decide which role should handle the message - # chosen_role_name = self.llm.ask(self.prompt_template.format(context)) - - # FIXME: 现在通过简单的字典决定流向，但之后还是应该有思考过程 - # The direction of flow is now determined by a simple dictionary, but there should still be a thought process afterwards - next_role_profile = self.role_directions[message.role] - # logger.debug(f"{next_role_profile}") - for _, role in roles.items(): - if next_role_profile == role.profile: - next_role = role - break - else: - logger.error(f"No available role can handle message: {message}.") - return - - # Find the chosen role and handle the message - return await next_role.handle(message) From 8a1237460eb1afd77be3d8db6d61adbcdcf271a2 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 19:27:11 +0800 Subject: [PATCH 32/49] remove useless fields --- metagpt/actions/action.py | 12 +----------- metagpt/actions/search_and_summarize.py | 3 +-- metagpt/roles/role.py | 2 +- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 7bb26ea91..1292b6684 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -26,22 +26,12 @@ class Action(ABC): self.llm = llm self.context = context self.prefix = "" # aask*时会加上prefix，作为system_message - self.profile = "" # FIXME: USELESS self.desc = "" # for skill manager self.nodes = ... - # Output, useless - # self.content = "" - # self.instruct_content = None - # self.env = None - - # def set_env(self, env): - # self.env = env - - def set_prefix(self, prefix, profile): + def set_prefix(self, prefix): """Set prefix for later usage""" self.prefix = prefix - self.profile = profile return self def __str__(self): diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 5e4cdaea0..a1d81bc65 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -130,8 +130,7 @@ class SearchAndSummarize(Action): system_prompt = [system_text] prompt = SEARCH_AND_SUMMARIZE_PROMPT.format( - # PREFIX = self.prefix, - ROLE=self.profile, + ROLE=self.prefix, CONTEXT=rsp, QUERY_HISTORY="\n".join([str(i) for i in context[:-1]]), QUERY=str(context[-1]), diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index e13bf454b..bf37a6637 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -146,7 +146,7 @@ class Role: self._actions = [] def _init_action_system_message(self, action: Action): - action.set_prefix(self._get_prefix(), self.profile) + action.set_prefix(self._get_prefix()) def _init_actions(self, actions): self._reset() From f0fd5ac59bd8be8e0083aa89a5d38d7cf3c3d639 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 21:17:02 +0800 Subject: [PATCH 33/49] refine a lot of code, fix pylint, use actionnode include ui, action _aask_v1, detail_mining, prepare_interview, etc. --- metagpt/actions/action.py | 48 +++----- metagpt/actions/action_node.py | 81 +++++--------- metagpt/actions/design_api.py | 10 +- metagpt/actions/detail_mining.py | 50 +++------ metagpt/actions/prepare_interview.py | 35 ++---- metagpt/actions/project_management.py | 10 +- metagpt/actions/write_prd.py | 8 +- metagpt/config.py | 2 +- metagpt/utils/get_template.py | 6 +- tests/metagpt/actions/test_detail_mining.py | 4 +- .../metagpt/actions/test_prepare_interview.py | 21 ++++ tests/metagpt/roles/ui_role.py | 104 +++++++++--------- 12 files changed, 163 insertions(+), 216 deletions(-) create mode 100644 tests/metagpt/actions/test_prepare_interview.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 1292b6684..5c5884e8b 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -6,19 +6,26 @@ @File : action.py """ +from __future__ import annotations + from abc import ABC from typing import Optional -from tenacity import retry, stop_after_attempt, wait_random_exponential - -from metagpt.actions.action_output import ActionOutput +from metagpt.actions.action_node import ActionNode from metagpt.llm import LLM -from metagpt.logs import logger -from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess -from metagpt.utils.common import OutputParser, general_after_log +from metagpt.schema import BaseContext class Action(ABC): + """Action abstract class, requiring all inheritors to provide a series of standard capabilities""" + + name: str + llm: LLM + context: dict | BaseContext | str | None + prefix: str + desc: str + node: ActionNode | None + def __init__(self, name: str = "", context=None, llm: LLM = None): self.name: str = name if llm is None: @@ -27,7 +34,7 @@ class Action(ABC): self.context = context self.prefix = "" # aask*时会加上prefix，作为system_message self.desc = "" # for skill manager - self.nodes = ... + self.node = None def set_prefix(self, prefix): """Set prefix for later usage""" @@ -47,33 +54,6 @@ class Action(ABC): system_msgs.append(self.prefix) return await self.llm.aask(prompt, system_msgs) - @retry( - wait=wait_random_exponential(min=1, max=60), - stop=stop_after_attempt(6), - after=general_after_log(logger), - ) - async def _aask_v1( - self, - prompt: str, - output_class_name: str, - output_data_mapping: dict, - system_msgs: Optional[list[str]] = None, - format="markdown", # compatible to original format - ) -> ActionOutput: - content = await self.llm.aask(prompt, system_msgs) - logger.debug(f"llm raw output:\n{content}") - output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) - - if format == "json": - parsed_data = llm_output_postprecess(output=content, schema=output_class.schema(), req_key="[/CONTENT]") - - else: # using markdown parser - parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) - - logger.debug(f"parsed_data:\n{parsed_data}") - instruct_content = output_class(**parsed_data) - return ActionOutput(content, instruct_content) - async def run(self, *args, **kwargs): """Run action""" raise NotImplementedError("The run method should be implemented in a subclass.") diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 6f1215920..0368d2df1 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -6,17 +6,15 @@ @File : action_node.py """ import json -import re -from typing import Any, Dict, List, Optional, Type +from typing import Dict, Generic, List, Optional, Type, TypeVar from pydantic import BaseModel, create_model, root_validator, validator from tenacity import retry, stop_after_attempt, wait_random_exponential -from metagpt.actions import ActionOutput from metagpt.llm import BaseGPTAPI from metagpt.logs import logger -from metagpt.utils.common import OutputParser -from metagpt.utils.custom_decoder import CustomDecoder +from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess +from metagpt.utils.common import OutputParser, general_after_log CONSTRAINT = """ - Language: Please use the same language as the user input. @@ -43,14 +41,17 @@ Fill in the above nodes based on the format example. """ -def dict_to_markdown(d, prefix="###", postfix="\n"): +def dict_to_markdown(d, prefix="-", postfix="\n"): markdown_str = "" for key, value in d.items(): markdown_str += f"{prefix} {key}: {value}{postfix}" return markdown_str -class ActionNode: +T = TypeVar("T") + + +class ActionNode(Generic[T]): """ActionNode is a tree of nodes.""" mode: str @@ -65,7 +66,7 @@ class ActionNode: expected_type: Type # such as str / int / float etc. # context: str # everything in the history. instruction: str # the instructions should be followed. - example: Any # example for In Context-Learning. + example: T # example for In Context-Learning. # Action Output content: str @@ -76,7 +77,7 @@ class ActionNode: key: str, expected_type: Type, instruction: str, - example: str, + example: T, content: str = "", children: dict[str, "ActionNode"] = None, ): @@ -148,29 +149,6 @@ class ActionNode: new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields) return new_class - @classmethod - def create_model_class_v2(cls, class_name: str, mapping: Dict[str, Type]): - """基于pydantic v2的模型动态生成，用来检验结果类型正确性，待验证""" - new_class = create_model(class_name, **mapping) - - @model_validator(mode="before") - def check_missing_fields(data): - required_fields = set(mapping.keys()) - missing_fields = required_fields - set(data.keys()) - if missing_fields: - raise ValueError(f"Missing fields: {missing_fields}") - return data - - @field_validator("*") - def check_name(v: Any, field: str) -> Any: - if field not in mapping.keys(): - raise ValueError(f"Unrecognized block: {field}") - return v - - new_class.__model_validator_check_missing_fields = classmethod(check_missing_fields) - new_class.__field_validator_check_name = classmethod(check_name) - return new_class - def create_children_class(self): """使用object内有的字段直接生成model_class""" class_name = f"{self.key}_AN" @@ -245,6 +223,7 @@ class ActionNode: """ # FIXME: json instruction会带来格式问题，如："Project name": "web_2048 # 项目名称使用下划线", + # compile example暂时不支持markdown self.instruction = self.compile_instruction(to="markdown", mode=mode) self.example = self.compile_example(to=to, tag="CONTENT", mode=mode) prompt = template.format( @@ -252,36 +231,32 @@ class ActionNode: ) return prompt - @retry(wait=wait_random_exponential(min=1, max=10), stop=stop_after_attempt(6)) + @retry( + wait=wait_random_exponential(min=1, max=60), + stop=stop_after_attempt(6), + after=general_after_log(logger), + ) async def _aask_v1( self, prompt: str, output_class_name: str, output_data_mapping: dict, system_msgs: Optional[list[str]] = None, - format="markdown", # compatible to original format - ) -> ActionOutput: + schema="markdown", # compatible to original format + ) -> (str, BaseModel): + """Use ActionOutput to wrap the output of aask""" content = await self.llm.aask(prompt, system_msgs) - logger.debug(content) - output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) - - if format == "json": - pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]" - matches = re.findall(pattern, content, re.DOTALL) - - for match in matches: - if match: - content = match - break - - parsed_data = CustomDecoder(strict=False).decode(content) + logger.debug(f"llm raw output:\n{content}") + output_class = self.create_model_class(output_class_name, output_data_mapping) + if schema == "json": + parsed_data = llm_output_postprecess(output=content, schema=output_class.schema(), req_key="[/CONTENT]") else: # using markdown parser parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) - logger.debug(parsed_data) + logger.debug(f"parsed_data:\n{parsed_data}") instruct_content = output_class(**parsed_data) - return ActionOutput(content, instruct_content) + return content, instruct_content def get(self, key): return self.instruct_content.dict()[key] @@ -302,9 +277,9 @@ class ActionNode: mapping = self.get_mapping(mode) class_name = f"{self.key}_AN" - output = await self._aask_v1(prompt, class_name, mapping, format=to) - self.content = output.content - self.instruct_content = output.instruct_content + content, scontent = await self._aask_v1(prompt, class_name, mapping, schema=to) + self.content = content + self.instruct_content = scontent return self async def fill(self, context, llm, to="json", mode="auto", strgy="simple"): diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 5a5f52de7..f757ca856 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -50,7 +50,7 @@ class WriteDesign(Action): "clearly and in detail." ) - async def run(self, with_messages, format=CONFIG.prompt_format): + async def run(self, with_messages, schema=CONFIG.prompt_schema): # Use `git diff` to identify which PRD documents have been modified in the `docs/prds` directory. prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) changed_prds = prds_file_repo.changed_files @@ -80,13 +80,13 @@ class WriteDesign(Action): # leaving room for global optimization in subsequent steps. return ActionOutput(content=changed_files.json(), instruct_content=changed_files) - async def _new_system_design(self, context, format=CONFIG.prompt_format): - node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format) + async def _new_system_design(self, context, schema=CONFIG.prompt_schema): + node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=schema) return node - async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): + async def _merge(self, prd_doc, system_design_doc, schema=CONFIG.prompt_schema): context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content) - node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=format) + node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=schema) system_design_doc.content = node.instruct_content.json(ensure_ascii=False) return system_design_doc diff --git a/metagpt/actions/detail_mining.py b/metagpt/actions/detail_mining.py index 5afcf52c6..0314d30dd 100644 --- a/metagpt/actions/detail_mining.py +++ b/metagpt/actions/detail_mining.py @@ -5,47 +5,31 @@ @Author : fisherdeng @File : detail_mining.py """ -from metagpt.actions import Action, ActionOutput +from metagpt.actions import Action +from metagpt.actions.action_node import ActionNode -PROMPT_TEMPLATE = """ -##TOPIC +CONTEXT_TEMPLATE = """ +## TOPIC {topic} -##RECORD +## RECORD {record} - -##Format example -{format_example} ------ - -Task: Refer to the "##TOPIC" (discussion objectives) and "##RECORD" (discussion records) to further inquire about the details that interest you, within a word limit of 150 words. -Special Note 1: Your intention is solely to ask questions without endorsing or negating any individual's viewpoints. -Special Note 2: This output should only include the topic "##OUTPUT". Do not add, remove, or modify the topic. Begin the output with '##OUTPUT', followed by an immediate line break, and then proceed to provide the content in the specified format as outlined in the "##Format example" section. -Special Note 3: The output should be in the same language as the input. """ -FORMAT_EXAMPLE = """ -## - -##OUTPUT -...(Please provide the specific details you would like to inquire about here.) - -## - -## -""" -OUTPUT_MAPPING = { - "OUTPUT": (str, ...), -} +QUESTIONS = ActionNode( + key="Questions", + expected_type=list[str], + instruction="Task: Refer to the context to further inquire about the details that interest you, within a word limit" + " of 150 words. Please provide the specific details you would like to inquire about here", + example=["1. What ...", "2. How ...", "3. ..."], +) class DetailMining(Action): - """This class allows LLM to further mine noteworthy details based on specific "##TOPIC"(discussion topic) and "##RECORD" (discussion records), thereby deepening the discussion.""" + """This class allows LLM to further mine noteworthy details based on specific "##TOPIC"(discussion topic) and + "##RECORD" (discussion records), thereby deepening the discussion.""" - def __init__(self, name="", context=None, llm=None): - super().__init__(name, context, llm) - - async def run(self, topic, record) -> ActionOutput: - prompt = PROMPT_TEMPLATE.format(topic=topic, record=record, format_example=FORMAT_EXAMPLE) - rsp = await self._aask_v1(prompt, "detail_mining", OUTPUT_MAPPING) + async def run(self, topic, record): + context = CONTEXT_TEMPLATE.format(topic=topic, record=record) + rsp = await QUESTIONS.fill(context=context, llm=self.llm) return rsp diff --git a/metagpt/actions/prepare_interview.py b/metagpt/actions/prepare_interview.py index b2704616e..7ed42d590 100644 --- a/metagpt/actions/prepare_interview.py +++ b/metagpt/actions/prepare_interview.py @@ -6,35 +6,18 @@ @File : prepare_interview.py """ from metagpt.actions import Action +from metagpt.actions.action_node import ActionNode -PROMPT_TEMPLATE = """ -# Context -{context} - -## Format example ---- -Q1: question 1 here -References: - - point 1 - - point 2 - -Q2: question 2 here... ---- - ------ -Role: You are an interviewer of our company who is well-knonwn in frontend or backend develop; +QUESTIONS = ActionNode( + key="Questions", + expected_type=list[str], + instruction="""Role: You are an interviewer of our company who is well-knonwn in frontend or backend develop; Requirement: Provide a list of questions for the interviewer to ask the interviewee, by reading the resume of the interviewee in the context. -Attention: Provide as markdown block as the format above, at least 10 questions. -""" - -# prepare for a interview +Attention: Provide as markdown block as the format above, at least 10 questions.""", + example=["1. What ...", "2. How ..."], +) class PrepareInterview(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) - async def run(self, context): - prompt = PROMPT_TEMPLATE.format(context=context) - question_list = await self._aask_v1(prompt) - return question_list + return await QUESTIONS.fill(context=context, llm=self.llm) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 1f14e7944..fe2c8d537 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -42,7 +42,7 @@ class WriteTasks(Action): def __init__(self, name="CreateTasks", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, with_messages, format=CONFIG.prompt_format): + async def run(self, with_messages, schema=CONFIG.prompt_schema): system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) changed_system_designs = system_design_file_repo.changed_files @@ -89,16 +89,16 @@ class WriteTasks(Action): await self._save_pdf(task_doc=task_doc) return task_doc - async def _run_new_tasks(self, context, format=CONFIG.prompt_format): - node = await PM_NODE.fill(context, self.llm, format) + async def _run_new_tasks(self, context, schema=CONFIG.prompt_schema): + node = await PM_NODE.fill(context, self.llm, schema) # prompt_template, format_example = get_template(templates, format) # prompt = prompt_template.format(context=context, format_example=format_example) # rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) return node - async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: + async def _merge(self, system_design_doc, task_doc, schema=CONFIG.prompt_schema) -> Document: context = NEW_REQ_TEMPLATE.format(context=system_design_doc.content, old_tasks=task_doc.content) - node = await PM_NODE.fill(context, self.llm, format) + node = await PM_NODE.fill(context, self.llm, schema) task_doc.content = node.instruct_content.json(ensure_ascii=False) return task_doc diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index adba7decb..1cf21dbb7 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -111,7 +111,7 @@ class WritePRD(Action): # optimization in subsequent steps. return ActionOutput(content=change_files.json(), instruct_content=change_files) - async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format) -> ActionOutput: + async def _run_new_requirement(self, requirements, schema=CONFIG.prompt_schema) -> ActionOutput: # sas = SearchAndSummarize() # # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) # rsp = "" @@ -121,7 +121,7 @@ class WritePRD(Action): # logger.info(rsp) project_name = CONFIG.project_name if CONFIG.project_name else "" context = CONTEXT_TEMPLATE.format(requirements=requirements, project_name=project_name) - node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, to=format) + node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, to=schema) await self._rename_workspace(node) return node @@ -130,11 +130,11 @@ class WritePRD(Action): node = await WP_IS_RELATIVE_NODE.fill(context, self.llm) return node.get("is_relative") == "YES" - async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: + async def _merge(self, new_requirement_doc, prd_doc, schema=CONFIG.prompt_schema) -> Document: if not CONFIG.project_name: CONFIG.project_name = Path(CONFIG.project_path).name prompt = NEW_REQ_TEMPLATE.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) - node = await WRITE_PRD_NODE.fill(context=prompt, llm=self.llm, to=format) + node = await WRITE_PRD_NODE.fill(context=prompt, llm=self.llm, to=schema) prd_doc.content = node.instruct_content.json(ensure_ascii=False) await self._rename_workspace(node) return prd_doc diff --git a/metagpt/config.py b/metagpt/config.py index 80a3a28f4..131854a56 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -143,7 +143,7 @@ class Config(metaclass=Singleton): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.repair_llm_output = self._get("REPAIR_LLM_OUTPUT", False) - self.prompt_format = self._get("PROMPT_FORMAT", "json") + self.prompt_schema = self._get("PROMPT_FORMAT", "json") self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) self._ensure_workspace_exists() diff --git a/metagpt/utils/get_template.py b/metagpt/utils/get_template.py index 86c1915f7..7e05e5d5e 100644 --- a/metagpt/utils/get_template.py +++ b/metagpt/utils/get_template.py @@ -8,10 +8,10 @@ from metagpt.config import CONFIG -def get_template(templates, format=CONFIG.prompt_format): - selected_templates = templates.get(format) +def get_template(templates, schema=CONFIG.prompt_schema): + selected_templates = templates.get(schema) if selected_templates is None: - raise ValueError(f"Can't find {format} in passed in templates") + raise ValueError(f"Can't find {schema} in passed in templates") # Extract the selected templates prompt_template = selected_templates["PROMPT_TEMPLATE"] diff --git a/tests/metagpt/actions/test_detail_mining.py b/tests/metagpt/actions/test_detail_mining.py index 891dca6ca..30bcf9dfb 100644 --- a/tests/metagpt/actions/test_detail_mining.py +++ b/tests/metagpt/actions/test_detail_mining.py @@ -19,5 +19,5 @@ async def test_detail_mining(): rsp = await detail_mining.run(topic=topic, record=record) logger.info(f"{rsp.content=}") - assert "##OUTPUT" in rsp.content - assert "蛋糕" in rsp.content + assert "Questions" in rsp.content + assert "1." in rsp.content diff --git a/tests/metagpt/actions/test_prepare_interview.py b/tests/metagpt/actions/test_prepare_interview.py new file mode 100644 index 000000000..7c32882e0 --- /dev/null +++ b/tests/metagpt/actions/test_prepare_interview.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/13 00:26 +@Author : fisherdeng +@File : test_detail_mining.py +""" +import pytest + +from metagpt.actions.prepare_interview import PrepareInterview +from metagpt.logs import logger + + +@pytest.mark.asyncio +async def test_prepare_interview(): + action = PrepareInterview() + rsp = await action.run("I just graduated and hope to find a job as a Python engineer") + logger.info(f"{rsp.content=}") + + assert "Questions" in rsp.content + assert "1." in rsp.content diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index 8ac799bf3..0932efa1f 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -10,6 +10,7 @@ from importlib import import_module from metagpt.actions import Action, ActionOutput, WritePRD # from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.action_node import ActionNode from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role @@ -17,44 +18,38 @@ from metagpt.schema import Message from metagpt.tools.sd_engine import SDEngine PROMPT_TEMPLATE = """ -# Context {context} -## Format example -{format_example} ------ -Role: You are a UserInterface Designer; the goal is to finish a UI design according to PRD, give a design description, and select specified elements and UI style. -Requirements: Based on the context, fill in the following missing information, provide detailed HTML and CSS code -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. - -## UI Design Description:Provide as Plain text, place the design objective here -## Selected Elements:Provide as Plain text, up to 5 specified elements, clear and simple -## HTML Layout:Provide as Plain text, use standard HTML code -## CSS Styles (styles.css):Provide as Plain text,use standard css code -## Anything UNCLEAR:Provide as Plain text. Try to clarify it. - +## Role +You are a UserInterface Designer; the goal is to finish a UI design according to PRD, give a design description, and select specified elements and UI style. """ -FORMAT_EXAMPLE = """ +UI_DESIGN_DESC = ActionNode( + key="UI Design Desc", + expected_type=str, + instruction="place the design objective here", + example="Snake games are classic and addictive games with simple yet engaging elements. Here are the main elements" + " commonly found in snake games", +) -## UI Design Description -```Snake games are classic and addictive games with simple yet engaging elements. Here are the main elements commonly found in snake games ``` +SELECTED_ELEMENTS = ActionNode( + key="Selected Elements", + expected_type=list[str], + instruction="up to 5 specified elements, clear and simple", + example=[ + "Game Grid: The game grid is a rectangular...", + "Snake: The player controls a snake that moves across the grid...", + "Food: Food items (often represented as small objects or differently colored blocks)", + "Score: The player's score increases each time the snake eats a piece of food. The longer the snake becomes, the higher the score.", + "Game Over: The game ends when the snake collides with itself or an obstacle. At this point, the player's final score is displayed, and they are given the option to restart the game.", + ], +) -## Selected Elements - -Game Grid: The game grid is a rectangular... - -Snake: The player controls a snake that moves across the grid... - -Food: Food items (often represented as small objects or differently colored blocks) - -Score: The player's score increases each time the snake eats a piece of food. The longer the snake becomes, the higher the score. - -Game Over: The game ends when the snake collides with itself or an obstacle. At this point, the player's final score is displayed, and they are given the option to restart the game. - - -## HTML Layout - +HTML_LAYOUT = ActionNode( + key="HTML Layout", + expected_type=str, + instruction="use standard HTML code", + example=""" @@ -71,9 +66,14 @@ Game Over: The game ends when the snake collides with itself or an obstacle. At +""", +) -## CSS Styles (styles.css) -body { +CSS_STYLES = ActionNode( + key="CSS Styles", + expected_type=str, + instruction="use standard css code", + example="""body { display: flex; justify-content: center; align-items: center; @@ -121,19 +121,25 @@ body { color: #ff0000; display: none; } +""", +) -## Anything UNCLEAR -There are no unclear points. +ANYTHING_UNCLEAR = ActionNode( + key="Anything UNCLEAR", + expected_type=str, + instruction="Mention any aspects of the project that are unclear and try to clarify them.", + example="...", +) -""" +NODES = [ + UI_DESIGN_DESC, + SELECTED_ELEMENTS, + HTML_LAYOUT, + CSS_STYLES, + ANYTHING_UNCLEAR, +] -OUTPUT_MAPPING = { - "UI Design Description": (str, ...), - "Selected Elements": (str, ...), - "HTML Layout": (str, ...), - "CSS Styles (styles.css)": (str, ...), - "Anything UNCLEAR": (str, ...), -} +UI_DESIGN_NODE = ActionNode.from_children("UI_DESIGN", NODES) def load_engine(func): @@ -223,10 +229,8 @@ class UIDesign(Action): css_file_path = save_dir / "ui_design.css" html_file_path = save_dir / "ui_design.html" - with open(css_file_path, "w") as css_file: - css_file.write(css_content) - with open(html_file_path, "w") as html_file: - html_file.write(html_content) + css_file_path.write_text(css_content) + html_file_path.write_text(html_content) async def run(self, requirements: list[Message], *args, **kwargs) -> ActionOutput: """Run the UI Design action.""" @@ -234,9 +238,9 @@ class UIDesign(Action): context = requirements[-1].content ui_design_draft = self.parse_requirement(context=context) # todo: parse requirements str - prompt = PROMPT_TEMPLATE.format(context=ui_design_draft, format_example=FORMAT_EXAMPLE) + prompt = PROMPT_TEMPLATE.format(context=ui_design_draft) logger.info(prompt) - ui_describe = await self._aask_v1(prompt, "ui_design", OUTPUT_MAPPING) + ui_describe = await UI_DESIGN_NODE.fill(prompt) logger.info(ui_describe.content) logger.info(ui_describe.instruct_content) css = self.parse_css_code(context=ui_describe.content) From 09e2f05a6a553c32cfdcdb53ec680d73acda1af2 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 21:24:08 +0800 Subject: [PATCH 34/49] refactor action_output and action_node --- metagpt/actions/action_node.py | 4 ++-- metagpt/actions/action_output.py | 26 +-------------------- metagpt/actions/write_prd.py | 2 +- metagpt/utils/serialize.py | 4 ++-- tests/metagpt/actions/test_action_output.py | 6 ++--- tests/metagpt/memory/test_memory_storage.py | 4 ++-- tests/metagpt/utils/test_serialize.py | 4 ++-- 7 files changed, 13 insertions(+), 37 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 0368d2df1..865cb2d32 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -6,7 +6,7 @@ @File : action_node.py """ import json -from typing import Dict, Generic, List, Optional, Type, TypeVar +from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar from pydantic import BaseModel, create_model, root_validator, validator from tenacity import retry, stop_after_attempt, wait_random_exponential @@ -127,7 +127,7 @@ class ActionNode(Generic[T]): return self.get_self_mapping() @classmethod - def create_model_class(cls, class_name: str, mapping: Dict[str, Type]): + def create_model_class(cls, class_name: str, mapping: Dict[str, Tuple[Type, Any]]): """基于pydantic v1的模型动态生成，用来检验结果类型正确性""" new_class = create_model(class_name, **mapping) diff --git a/metagpt/actions/action_output.py b/metagpt/actions/action_output.py index 25326d43b..6be8dac50 100644 --- a/metagpt/actions/action_output.py +++ b/metagpt/actions/action_output.py @@ -6,9 +6,7 @@ @File : action_output """ -from typing import Dict, Type - -from pydantic import BaseModel, create_model, root_validator, validator +from pydantic import BaseModel class ActionOutput: @@ -18,25 +16,3 @@ class ActionOutput: def __init__(self, content: str, instruct_content: BaseModel): self.content = content self.instruct_content = instruct_content - - @classmethod - def create_model_class(cls, class_name: str, mapping: Dict[str, Type]): - new_class = create_model(class_name, **mapping) - - @validator("*", allow_reuse=True) - def check_name(v, field): - if field.name not in mapping.keys(): - raise ValueError(f"Unrecognized block: {field.name}") - return v - - @root_validator(pre=True, allow_reuse=True) - def check_missing_fields(values): - required_fields = set(mapping.keys()) - missing_fields = required_fields - set(values.keys()) - if missing_fields: - raise ValueError(f"Missing fields: {missing_fields}") - return values - - new_class.__validator_check_name = classmethod(check_name) - new_class.__root_validator_check_missing_fields = classmethod(check_missing_fields) - return new_class diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 1cf21dbb7..23925ff10 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -67,7 +67,7 @@ class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput | Message: + async def run(self, with_messages, schema=CONFIG.prompt_schema, *args, **kwargs) -> ActionOutput | Message: # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are # related to the PRD. If they are related, rewrite the PRD. docs_file_repo = CONFIG.git_repo.new_file_repository(relative_path=DOCS_FILE_REPO) diff --git a/metagpt/utils/serialize.py b/metagpt/utils/serialize.py index 124176fcb..5e52846e1 100644 --- a/metagpt/utils/serialize.py +++ b/metagpt/utils/serialize.py @@ -6,7 +6,7 @@ import copy import pickle from typing import Dict, List -from metagpt.actions.action_output import ActionOutput +from metagpt.actions.action_node import ActionNode from metagpt.schema import Message @@ -60,7 +60,7 @@ def deserialize_message(message_ser: str) -> Message: message = pickle.loads(message_ser) if message.instruct_content: ic = message.instruct_content - ic_obj = ActionOutput.create_model_class(class_name=ic["class"], mapping=ic["mapping"]) + ic_obj = ActionNode.create_model_class(class_name=ic["class"], mapping=ic["mapping"]) ic_new = ic_obj(**ic["value"]) message.instruct_content = ic_new diff --git a/tests/metagpt/actions/test_action_output.py b/tests/metagpt/actions/test_action_output.py index ef8e239bd..f1765cb03 100644 --- a/tests/metagpt/actions/test_action_output.py +++ b/tests/metagpt/actions/test_action_output.py @@ -7,7 +7,7 @@ """ from typing import List, Tuple -from metagpt.actions import ActionOutput +from metagpt.actions.action_node import ActionNode t_dict = { "Required Python third-party packages": '"""\nflask==1.1.2\npygame==2.0.1\n"""\n', @@ -37,12 +37,12 @@ WRITE_TASKS_OUTPUT_MAPPING = { def test_create_model_class(): - test_class = ActionOutput.create_model_class("test_class", WRITE_TASKS_OUTPUT_MAPPING) + test_class = ActionNode.create_model_class("test_class", WRITE_TASKS_OUTPUT_MAPPING) assert test_class.__name__ == "test_class" def test_create_model_class_with_mapping(): - t = ActionOutput.create_model_class("test_class_1", WRITE_TASKS_OUTPUT_MAPPING) + t = ActionNode.create_model_class("test_class_1", WRITE_TASKS_OUTPUT_MAPPING) t1 = t(**t_dict) value = t1.dict()["Task list"] assert value == ["game.py", "app.py", "static/css/styles.css", "static/js/script.js", "templates/index.html"] diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index c67ca689f..7b74eb512 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -8,7 +8,7 @@ from typing import List from metagpt.actions import UserRequirement, WritePRD -from metagpt.actions.action_output import ActionOutput +from metagpt.actions.action_node import ActionNode from metagpt.memory.memory_storage import MemoryStorage from metagpt.schema import Message @@ -42,7 +42,7 @@ def test_idea_message(): def test_actionout_message(): out_mapping = {"field1": (str, ...), "field2": (List[str], ...)} out_data = {"field1": "field1 value", "field2": ["field2 value1", "field2 value2"]} - ic_obj = ActionOutput.create_model_class("prd", out_mapping) + ic_obj = ActionNode.create_model_class("prd", out_mapping) role_id = "UTUser2(Architect)" content = "The user has requested the creation of a command-line interface (CLI) snake game" diff --git a/tests/metagpt/utils/test_serialize.py b/tests/metagpt/utils/test_serialize.py index ffa34866c..f027d53f8 100644 --- a/tests/metagpt/utils/test_serialize.py +++ b/tests/metagpt/utils/test_serialize.py @@ -7,7 +7,7 @@ from typing import List, Tuple from metagpt.actions import WritePRD -from metagpt.actions.action_output import ActionOutput +from metagpt.actions.action_node import ActionNode from metagpt.schema import Message from metagpt.utils.serialize import ( actionoutout_schema_to_mapping, @@ -54,7 +54,7 @@ def test_actionoutout_schema_to_mapping(): def test_serialize_and_deserialize_message(): out_mapping = {"field1": (str, ...), "field2": (List[str], ...)} out_data = {"field1": "field1 value", "field2": ["field2 value1", "field2 value2"]} - ic_obj = ActionOutput.create_model_class("prd", out_mapping) + ic_obj = ActionNode.create_model_class("prd", out_mapping) message = Message( content="prd demand", instruct_content=ic_obj(**out_data), role="user", cause_by=WritePRD From 33c58d97fef317afba757ba04ece00fd1830130d Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 21:32:52 +0800 Subject: [PATCH 35/49] refine code --- metagpt/actions/action_node.py | 2 +- metagpt/actions/write_prd_an.py | 8 ++++---- metagpt/provider/postprecess/base_postprecess_plugin.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 865cb2d32..790069369 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -232,7 +232,7 @@ class ActionNode(Generic[T]): return prompt @retry( - wait=wait_random_exponential(min=1, max=60), + wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6), after=general_after_log(logger), ) diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index d96c0aeac..edd94a463 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -47,7 +47,7 @@ PRODUCT_GOALS = ActionNode( USER_STORIES = ActionNode( key="User Stories", expected_type=list[str], - instruction="Provide up to five scenario-based user stories.", + instruction="Provide up to 3 to 5 scenario-based user stories.", example=[ "As a user, I want to be able to choose difficulty levels", "As a player, I want to see my score after each game", @@ -57,7 +57,7 @@ USER_STORIES = ActionNode( COMPETITIVE_ANALYSIS = ActionNode( key="Competitive Analysis", expected_type=list[str], - instruction="Provide analyses for up to seven competitive products.", + instruction="Provide 5 to 7 competitive products.", example=["Python Snake Game: Simple interface, lacks advanced features"], ) @@ -92,8 +92,8 @@ REQUIREMENT_ANALYSIS = ActionNode( REQUIREMENT_POOL = ActionNode( key="Requirement Pool", expected_type=list[list[str]], - instruction="List down the requirements with their priority (P0, P1, P2).", - example=[["P0", "..."], ["P1", "..."]], + instruction="List down the top-5 requirements with their priority (P0, P1, P2).", + example=[["P0", "The main code ..."], ["P0", "The game algorithm ..."]], ) UI_DESIGN_DRAFT = ActionNode( diff --git a/metagpt/provider/postprecess/base_postprecess_plugin.py b/metagpt/provider/postprecess/base_postprecess_plugin.py index 0d1cfbb11..721476507 100644 --- a/metagpt/provider/postprecess/base_postprecess_plugin.py +++ b/metagpt/provider/postprecess/base_postprecess_plugin.py @@ -44,7 +44,7 @@ class BasePostPrecessPlugin(object): def run_retry_parse_json_text(self, content: str) -> Union[dict, list]: """inherited class can re-implement the function""" - logger.info(f"extracted json CONTENT from output:\n{content}") + logger.debug(f"extracted json CONTENT from output:\n{content}") parsed_data = retry_parse_json_text(output=content) # should use output=content return parsed_data From 62f34db137dcd73b965e613497ca1dd2df1ddcd9 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 23:53:04 +0800 Subject: [PATCH 36/49] refine code. move azure tts to tool, refactor actions --- metagpt/actions/__init__.py | 2 - metagpt/actions/action.py | 5 ++- metagpt/actions/analyze_dep_libs.py | 37 ------------------- metagpt/actions/design_filenames.py | 30 --------------- ...detail_mining.py => generate_questions.py} | 18 ++------- metagpt/schema.py | 3 +- metagpt/{actions => tools}/azure_tts.py | 19 ++++------ tests/metagpt/actions/test_azure_tts.py | 4 +- tests/metagpt/actions/test_detail_mining.py | 20 ++++++---- 9 files changed, 32 insertions(+), 106 deletions(-) delete mode 100644 metagpt/actions/analyze_dep_libs.py delete mode 100644 metagpt/actions/design_filenames.py rename metagpt/actions/{detail_mining.py => generate_questions.py} (69%) rename metagpt/{actions => tools}/azure_tts.py (65%) diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index 79ff94b3e..c34c72ed2 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -13,7 +13,6 @@ from metagpt.actions.add_requirement import UserRequirement from metagpt.actions.debug_error import DebugError from metagpt.actions.design_api import WriteDesign from metagpt.actions.design_api_review import DesignReview -from metagpt.actions.design_filenames import DesignFilenames from metagpt.actions.project_management import AssignTasks, WriteTasks from metagpt.actions.research import CollectLinks, WebBrowseAndSummarize, ConductResearch from metagpt.actions.run_code import RunCode @@ -33,7 +32,6 @@ class ActionType(Enum): WRITE_PRD_REVIEW = WritePRDReview WRITE_DESIGN = WriteDesign DESIGN_REVIEW = DesignReview - DESIGN_FILENAMES = DesignFilenames WRTIE_CODE = WriteCode WRITE_CODE_REVIEW = WriteCodeReview WRITE_TEST = WriteTest diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 5c5884e8b..a3a9c0195 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -13,7 +13,7 @@ from typing import Optional from metagpt.actions.action_node import ActionNode from metagpt.llm import LLM -from metagpt.schema import BaseContext +from metagpt.schema import CodingContext, CodeSummarizeContext, TestingContext, RunCodeContext class Action(ABC): @@ -21,7 +21,8 @@ class Action(ABC): name: str llm: LLM - context: dict | BaseContext | str | None + # FIXME: simplify context + context: dict | CodingContext | CodeSummarizeContext | TestingContext | RunCodeContext | str | None prefix: str desc: str node: ActionNode | None diff --git a/metagpt/actions/analyze_dep_libs.py b/metagpt/actions/analyze_dep_libs.py deleted file mode 100644 index 53d40200a..000000000 --- a/metagpt/actions/analyze_dep_libs.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/19 12:01 -@Author : alexanderwu -@File : analyze_dep_libs.py -""" - -from metagpt.actions import Action - -PROMPT = """You are an AI developer, trying to write a program that generates code for users based on their intentions. - -For the user's prompt: - ---- -The API is: {prompt} ---- - -We decide the generated files are: {filepaths_string} - -Now that we have a file list, we need to understand the shared dependencies they have. -Please list and briefly describe the shared contents between the files we are generating, including exported variables, -data patterns, id names of all DOM elements that javascript functions will use, message names and function names. -Focus only on the names of shared dependencies, do not add any other explanations. -""" - - -class AnalyzeDepLibs(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) - self.desc = "Analyze the runtime dependencies of the program based on the context" - - async def run(self, requirement, filepaths_string): - # prompt = f"Below is the product requirement document (PRD):\n\n{prd}\n\n{PROMPT}" - prompt = PROMPT.format(prompt=requirement, filepaths_string=filepaths_string) - design_filenames = await self._aask(prompt) - return design_filenames diff --git a/metagpt/actions/design_filenames.py b/metagpt/actions/design_filenames.py deleted file mode 100644 index ffa171d7b..000000000 --- a/metagpt/actions/design_filenames.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/19 11:50 -@Author : alexanderwu -@File : design_filenames.py -""" -from metagpt.actions import Action -from metagpt.logs import logger - -PROMPT = """You are an AI developer, trying to write a program that generates code for users based on their intentions. -When given their intentions, provide a complete and exhaustive list of file paths needed to write the program for the user. -Only list the file paths you will write and return them as a Python string list. -Do not add any other explanations, just return a Python string list.""" - - -class DesignFilenames(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) - self.desc = ( - "Based on the PRD, consider system design, and carry out the basic design of the corresponding " - "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail." - ) - - async def run(self, prd): - prompt = f"The following is the Product Requirement Document (PRD):\n\n{prd}\n\n{PROMPT}" - design_filenames = await self._aask(prompt) - logger.debug(prompt) - logger.debug(design_filenames) - return design_filenames diff --git a/metagpt/actions/detail_mining.py b/metagpt/actions/generate_questions.py similarity index 69% rename from metagpt/actions/detail_mining.py rename to metagpt/actions/generate_questions.py index 0314d30dd..c38c463bc 100644 --- a/metagpt/actions/detail_mining.py +++ b/metagpt/actions/generate_questions.py @@ -3,19 +3,11 @@ """ @Time : 2023/9/12 17:45 @Author : fisherdeng -@File : detail_mining.py +@File : generate_questions.py """ from metagpt.actions import Action from metagpt.actions.action_node import ActionNode -CONTEXT_TEMPLATE = """ -## TOPIC -{topic} - -## RECORD -{record} -""" - QUESTIONS = ActionNode( key="Questions", expected_type=list[str], @@ -25,11 +17,9 @@ QUESTIONS = ActionNode( ) -class DetailMining(Action): +class GenerateQuestions(Action): """This class allows LLM to further mine noteworthy details based on specific "##TOPIC"(discussion topic) and "##RECORD" (discussion records), thereby deepening the discussion.""" - async def run(self, topic, record): - context = CONTEXT_TEMPLATE.format(topic=topic, record=record) - rsp = await QUESTIONS.fill(context=context, llm=self.llm) - return rsp + async def run(self, context): + return await QUESTIONS.fill(context=context, llm=self.llm) diff --git a/metagpt/schema.py b/metagpt/schema.py index aacc2cebb..d2f8d33e6 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -18,6 +18,7 @@ import asyncio import json import os.path import uuid +from abc import ABC from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError from pathlib import Path @@ -265,7 +266,7 @@ class MessageQueue: T = TypeVar("T", bound="BaseModel") -class BaseContext(BaseModel): +class BaseContext(BaseModel, ABC): @classmethod @handle_exception def loads(cls: Type[T], val: str) -> Optional[T]: diff --git a/metagpt/actions/azure_tts.py b/metagpt/tools/azure_tts.py similarity index 65% rename from metagpt/actions/azure_tts.py rename to metagpt/tools/azure_tts.py index daa3f6892..e59d98016 100644 --- a/metagpt/actions/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -7,19 +7,16 @@ """ from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer -from metagpt.actions.action import Action -from metagpt.config import Config +from metagpt.config import CONFIG -class AzureTTS(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) - self.config = Config() +class AzureTTS: + """https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles""" - # Parameters reference: https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles - def synthesize_speech(self, lang, voice, role, text, output_file): - subscription_key = self.config.get("AZURE_TTS_SUBSCRIPTION_KEY") - region = self.config.get("AZURE_TTS_REGION") + @classmethod + def synthesize_speech(cls, lang, voice, role, text, output_file): + subscription_key = CONFIG.get("AZURE_TTS_SUBSCRIPTION_KEY") + region = CONFIG.get("AZURE_TTS_REGION") speech_config = SpeechConfig(subscription=subscription_key, region=region) speech_config.speech_synthesis_voice_name = voice @@ -41,5 +38,5 @@ class AzureTTS(Action): if __name__ == "__main__": - azure_tts = AzureTTS("azure_tts") + azure_tts = AzureTTS() azure_tts.synthesize_speech("zh-CN", "zh-CN-YunxiNeural", "Boy", "Hello, I am Kaka", "output.wav") diff --git a/tests/metagpt/actions/test_azure_tts.py b/tests/metagpt/actions/test_azure_tts.py index bcafe10f5..9995e9691 100644 --- a/tests/metagpt/actions/test_azure_tts.py +++ b/tests/metagpt/actions/test_azure_tts.py @@ -5,11 +5,11 @@ @Author : alexanderwu @File : test_azure_tts.py """ -from metagpt.actions.azure_tts import AzureTTS +from metagpt.tools.azure_tts import AzureTTS def test_azure_tts(): - azure_tts = AzureTTS("azure_tts") + azure_tts = AzureTTS() azure_tts.synthesize_speech("zh-CN", "zh-CN-YunxiNeural", "Boy", "你好，我是卡卡", "output.wav") # 运行需要先配置 SUBSCRIPTION_KEY diff --git a/tests/metagpt/actions/test_detail_mining.py b/tests/metagpt/actions/test_detail_mining.py index 30bcf9dfb..a178ec840 100644 --- a/tests/metagpt/actions/test_detail_mining.py +++ b/tests/metagpt/actions/test_detail_mining.py @@ -3,20 +3,26 @@ """ @Time : 2023/9/13 00:26 @Author : fisherdeng -@File : test_detail_mining.py +@File : test_generate_questions.py """ import pytest -from metagpt.actions.detail_mining import DetailMining +from metagpt.actions.generate_questions import GenerateQuestions from metagpt.logs import logger +context = """ +## topic +如何做一个生日蛋糕 + +## record +我认为应该先准备好材料，然后再开始做蛋糕。 +""" + @pytest.mark.asyncio -async def test_detail_mining(): - topic = "如何做一个生日蛋糕" - record = "我认为应该先准备好材料，然后再开始做蛋糕。" - detail_mining = DetailMining("detail_mining") - rsp = await detail_mining.run(topic=topic, record=record) +async def test_generate_questions(): + detail_mining = GenerateQuestions() + rsp = await detail_mining.run(context) logger.info(f"{rsp.content=}") assert "Questions" in rsp.content From 0f78d4ea51d6e7d579dc7340e9b7e2039d0f5aa2 Mon Sep 17 00:00:00 2001 From: geekan Date: Tue, 19 Dec 2023 23:58:18 +0800 Subject: [PATCH 37/49] refine code --- metagpt/actions/action_node.py | 52 +++++++++++++++++----------------- metagpt/actions/design_api.py | 4 +-- metagpt/actions/write_prd.py | 4 +-- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 790069369..092dd5755 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -112,15 +112,15 @@ class ActionNode(Generic[T]): obj.add_children(nodes) return obj - def get_children_mapping(self) -> Dict[str, Type]: + def get_children_mapping(self) -> Dict[str, Tuple[Type, Any]]: """获得子ActionNode的字典，以key索引""" return {k: (v.expected_type, ...) for k, v in self.children.items()} - def get_self_mapping(self) -> Dict[str, Type]: + def get_self_mapping(self) -> Dict[str, Tuple[Type, Any]]: """get self key: type mapping""" return {self.key: (self.expected_type, ...)} - def get_mapping(self, mode="children") -> Dict[str, Type]: + def get_mapping(self, mode="children") -> Dict[str, Tuple[Type, Any]]: """get key: type mapping under mode""" if mode == "children" or (mode == "auto" and self.children): return self.get_children_mapping() @@ -175,46 +175,46 @@ class ActionNode(Generic[T]): return node_dict # 遍历子节点并递归调用 to_dict 方法 - for child_key, child_node in self.children.items(): + for _, child_node in self.children.items(): node_dict.update(child_node.to_dict(format_func)) return node_dict - def compile_to(self, i: Dict, to) -> str: - if to == "json": + def compile_to(self, i: Dict, schema) -> str: + if schema == "json": return json.dumps(i, indent=4) - elif to == "markdown": + elif schema == "markdown": return dict_to_markdown(i) else: return str(i) - def tagging(self, text, to, tag="") -> str: + def tagging(self, text, schema, tag="") -> str: if not tag: return text - if to == "json": + if schema == "json": return f"[{tag}]\n" + text + f"\n[/{tag}]" else: return f"[{tag}]\n" + text + f"\n[/{tag}]" - def _compile_f(self, to, mode, tag, format_func) -> str: + def _compile_f(self, schema, mode, tag, format_func) -> str: nodes = self.to_dict(format_func=format_func, mode=mode) - text = self.compile_to(nodes, to) - return self.tagging(text, to, tag) + text = self.compile_to(nodes, schema) + return self.tagging(text, schema, tag) - def compile_instruction(self, to="raw", mode="children", tag="") -> str: + def compile_instruction(self, schema="raw", mode="children", tag="") -> str: """compile to raw/json/markdown template with all/root/children nodes""" format_func = lambda i: f"{i.expected_type} # {i.instruction}" - return self._compile_f(to, mode, tag, format_func) + return self._compile_f(schema, mode, tag, format_func) - def compile_example(self, to="raw", mode="children", tag="") -> str: + def compile_example(self, schema="raw", mode="children", tag="") -> str: """compile to raw/json/markdown examples with all/root/children nodes""" # 这里不能使用f-string，因为转译为str后再json.dumps会额外加上引号，无法作为有效的example # 错误示例："File list": "['main.py', 'const.py', 'game.py']", 注意这里值不是list，而是str format_func = lambda i: i.example - return self._compile_f(to, mode, tag, format_func) + return self._compile_f(schema, mode, tag, format_func) - def compile(self, context, to="json", mode="children", template=SIMPLE_TEMPLATE) -> str: + def compile(self, context, schema="json", mode="children", template=SIMPLE_TEMPLATE) -> str: """ mode: all/root/children mode="children": 编译所有子节点为一个统一模板，包括instruction与example @@ -224,8 +224,8 @@ class ActionNode(Generic[T]): # FIXME: json instruction会带来格式问题，如："Project name": "web_2048 # 项目名称使用下划线", # compile example暂时不支持markdown - self.instruction = self.compile_instruction(to="markdown", mode=mode) - self.example = self.compile_example(to=to, tag="CONTENT", mode=mode) + self.instruction = self.compile_instruction(schema="markdown", mode=mode) + self.example = self.compile_example(schema=schema, tag="CONTENT", mode=mode) prompt = template.format( context=context, example=self.example, instruction=self.instruction, constraint=CONSTRAINT ) @@ -272,22 +272,22 @@ class ActionNode(Generic[T]): def set_context(self, context): self.set_recursive("context", context) - async def simple_fill(self, to, mode): - prompt = self.compile(context=self.context, to=to, mode=mode) + async def simple_fill(self, schema, mode): + prompt = self.compile(context=self.context, schema=schema, mode=mode) mapping = self.get_mapping(mode) class_name = f"{self.key}_AN" - content, scontent = await self._aask_v1(prompt, class_name, mapping, schema=to) + content, scontent = await self._aask_v1(prompt, class_name, mapping, schema=schema) self.content = content self.instruct_content = scontent return self - async def fill(self, context, llm, to="json", mode="auto", strgy="simple"): + async def fill(self, context, llm, schema="json", mode="auto", strgy="simple"): """Fill the node(s) with mode. :param context: Everything we should know when filling node. :param llm: Large Language Model with pre-defined system message. - :param to: json/markdown, determine example and output format. + :param schema: json/markdown, determine example and output format. - json: it's easy to open source LLM with json format - markdown: when generating code, markdown is always better :param mode: auto/children/root @@ -303,12 +303,12 @@ class ActionNode(Generic[T]): self.set_context(context) if strgy == "simple": - return await self.simple_fill(to, mode) + return await self.simple_fill(schema, mode) elif strgy == "complex": # 这里隐式假设了拥有children tmp = {} for _, i in self.children.items(): - child = await i.simple_fill(to, mode) + child = await i.simple_fill(schema, mode) tmp.update(child.instruct_content.dict()) cls = self.create_children_class() self.instruct_content = cls(**tmp) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index f757ca856..548725fde 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -81,12 +81,12 @@ class WriteDesign(Action): return ActionOutput(content=changed_files.json(), instruct_content=changed_files) async def _new_system_design(self, context, schema=CONFIG.prompt_schema): - node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=schema) + node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, schema=schema) return node async def _merge(self, prd_doc, system_design_doc, schema=CONFIG.prompt_schema): context = NEW_REQ_TEMPLATE.format(old_design=system_design_doc.content, context=prd_doc.content) - node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, to=schema) + node = await DESIGN_API_NODE.fill(context=context, llm=self.llm, schema=schema) system_design_doc.content = node.instruct_content.json(ensure_ascii=False) return system_design_doc diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 23925ff10..7c160fa89 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -121,7 +121,7 @@ class WritePRD(Action): # logger.info(rsp) project_name = CONFIG.project_name if CONFIG.project_name else "" context = CONTEXT_TEMPLATE.format(requirements=requirements, project_name=project_name) - node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, to=schema) + node = await WRITE_PRD_NODE.fill(context=context, llm=self.llm, schema=schema) await self._rename_workspace(node) return node @@ -134,7 +134,7 @@ class WritePRD(Action): if not CONFIG.project_name: CONFIG.project_name = Path(CONFIG.project_path).name prompt = NEW_REQ_TEMPLATE.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) - node = await WRITE_PRD_NODE.fill(context=prompt, llm=self.llm, to=schema) + node = await WRITE_PRD_NODE.fill(context=prompt, llm=self.llm, schema=schema) prd_doc.content = node.instruct_content.json(ensure_ascii=False) await self._rename_workspace(node) return prd_doc From d0382b0ba7dfa69c7aafb7f6619c81531637d728 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 00:34:57 +0800 Subject: [PATCH 38/49] refine devcontainer README --- .devcontainer/README.md | 41 ++++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/.devcontainer/README.md b/.devcontainer/README.md index dd088aab1..be692c14d 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -1,39 +1,34 @@ -# Dev container +# Dev Container -This project includes a [dev container](https://containers.dev/), which lets you use a container as a full-featured dev environment. +This project includes a [Dev Container](https://containers.dev/), offering you a comprehensive and fully-featured development environment within a container. By leveraging the Dev Container configuration in this folder, you can seamlessly build and initiate MetaGPT locally. For detailed information, please refer to the main README in the home directory. -You can use the dev container configuration in this folder to build and start running MetaGPT locally! For more, refer to the main README under the home directory. -You can use it in [GitHub Codespaces](https://github.com/features/codespaces) or the [VS Code Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). +You can utilize this Dev Container in [GitHub Codespaces](https://github.com/features/codespaces) or with the [VS Code Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers). ## GitHub Codespaces -Open in GitHub Codespaces +[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/geekan/MetaGPT) -You may use the button above to open this repo in a Codespace +Click the button above to open this repository in a Codespace. For additional information, refer to the [GitHub documentation on creating a Codespace](https://docs.github.com/en/free-pro-team@latest/github/developing-online-with-codespaces/creating-a-codespace#creating-a-codespace). -For more info, check out the [GitHub documentation](https://docs.github.com/en/free-pro-team@latest/github/developing-online-with-codespaces/creating-a-codespace#creating-a-codespace). - ## VS Code Dev Containers -Open in Dev Containers +[![Open in Dev Containers](https://img.shields.io/static/v1?label=Dev%20Containers&message=Open&color=blue&logo=visualstudiocode)](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/geekan/MetaGPT) -Note: If you click this link you will open the main repo and not your local cloned repo, you can use this link and replace with your username and cloned repo name: -https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/geekan/MetaGPT +Note: Clicking the link above opens the main repository. To open your local cloned repository, replace the URL with your username and cloned repository's name: `https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com//` +If you have VS Code and Docker installed, use the button above to get started. This will prompt VS Code to install the Dev Containers extension if it's not already installed, clone the source code into a container volume, and set up a dev container for you. -If you already have VS Code and Docker installed, you can use the button above to get started. This will cause VS Code to automatically install the Dev Containers extension if needed, clone the source code into a container volume, and spin up a dev container for use. +Alternatively, follow these steps to open this repository in a container using the VS Code Dev Containers extension: -You can also follow these steps to open this repo in a container using the VS Code Dev Containers extension: +1. For first-time users of a development container, ensure your system meets the prerequisites (e.g., Docker installation) as outlined in the [getting started steps](https://aka.ms/vscode-remote/containers/getting-started). -1. If this is your first time using a development container, please ensure your system meets the pre-reqs (i.e. have Docker installed) in the [getting started steps](https://aka.ms/vscode-remote/containers/getting-started). - -2. Open a locally cloned copy of the code: - - - Fork and Clone this repository to your local filesystem. +2. To open a locally cloned copy of the code: + - Fork and clone this repository to your local file system. - Press F1 and select the **Dev Containers: Open Folder in Container...** command. - - Select the cloned copy of this folder, wait for the container to start, and try things out! + - Choose the cloned folder, wait for the container to initialize, and start exploring! -You can learn more in the [Dev Containers documentation](https://code.visualstudio.com/docs/devcontainers/containers). +Learn more in the [VS Code Dev Containers documentation](https://code.visualstudio.com/docs/devcontainers/containers). -## Tips and tricks +## Tips and Tricks -* If you are working with the same repository folder in a container and Windows, you'll want consistent line endings (otherwise you may see hundreds of changes in the SCM view). The `.gitattributes` file in the root of this repo will disable line ending conversion and should prevent this. See [tips and tricks](https://code.visualstudio.com/docs/devcontainers/tips-and-tricks#_resolving-git-line-ending-issues-in-containers-resulting-in-many-modified-files) for more info. -* If you'd like to review the contents of the image used in this dev container, you can check it out in the [devcontainers/images](https://github.com/devcontainers/images/tree/main/src/python) repo. +* When working with the same repository folder in both a container and on Windows, it's crucial to have consistent line endings to avoid numerous changes in the SCM view. The `.gitattributes` file in the root of this repository disables line ending conversion, helping to prevent this issue. For more information, see [resolving git line ending issues in containers](https://code.visualstudio.com/docs/devcontainers/tips-and-tricks#_resolving-git-line-ending-issues-in-containers-resulting-in-many-modified-files). + +* If you're curious about the contents of the image used in this Dev Container, you can review it in the [devcontainers/images](https://github.com/devcontainers/images/tree/main/src/python) repository. From 1a62148dc6ea684a9dc0da372dc5c1ba3ac785a9 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 00:35:15 +0800 Subject: [PATCH 39/49] add proper space --- .devcontainer/postCreateCommand.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index 46788e306..3901193cd 100644 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -4,4 +4,4 @@ sudo npm install -g @mermaid-js/mermaid-cli # Step 2: Ensure that Python 3.9+ is installed on your system. You can check this by using: python --version -pip install -e. \ No newline at end of file +pip install -e . \ No newline at end of file From 6b235e536e6d5b2590db97cdcd4aece779227c13 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 00:39:35 +0800 Subject: [PATCH 40/49] .gitattributes: ensure lf --- .gitattributes | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.gitattributes b/.gitattributes index 32555a806..7f1424434 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,29 @@ +# HTML code is incorrectly calculated into statistics, so ignore them *.html linguist-detectable=false +# Auto detect text files and perform LF normalization +* text=auto eol=lf + +# Ensure shell scripts use LF (Linux style) line endings on Windows +*.sh text eol=lf + +# Treat specific binary files as binary and prevent line ending conversion +*.png binary +*.jpg binary +*.gif binary +*.ico binary + +# Preserve original line endings for specific document files +*.doc text eol=crlf +*.docx text eol=crlf +*.pdf binary + +# Ensure source code and script files use LF line endings +*.py text eol=lf +*.js text eol=lf +*.html text eol=lf +*.css text eol=lf + +# Specify custom diff driver for specific file types +*.md diff=markdown +*.json diff=json From efebc07e54374accd65c7a82c2c10fb4b1dfdb0a Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 00:47:28 +0800 Subject: [PATCH 41/49] refine .gitignore and .pre-commit-config.yaml --- .gitignore | 8 +------- .pre-commit-config.yaml | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 0ac318ff5..c12506b0e 100644 --- a/.gitignore +++ b/.gitignore @@ -144,24 +144,18 @@ cython_debug/ allure-report allure-results -# idea +# idea / vscode / macos .idea .DS_Store .vscode -log.txt -docs/scripts/set_env.sh key.yaml -output.json data -data/output_add.json data.ms examples/nb/ .chroma *~$* workspace/* -*.mmd tmp -output.wav metagpt/roles/idea_agent.py .aider* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b1892a709..338f832ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ default_stages: [ commit ] # Install # 1. pip install pre-commit -# 2. pre-commit install(the first time you download the repo, it will be cached for future use) +# 2. pre-commit install repos: - repo: https://github.com/pycqa/isort rev: 5.11.5 From 3b7c2e48599b9837894de766eb7f6bb275752667 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 00:49:08 +0800 Subject: [PATCH 42/49] updating time of license --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 5b0c000cd..67460e101 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License -Copyright (c) Chenglin Wu +Copyright (c) 2023 Chenglin Wu Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 394055d7e6380b05f28ffaebf53b7ae50c9d79a6 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 00:53:36 +0800 Subject: [PATCH 43/49] align ruff.toml with black --- ruff.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index 7835865e0..21de5ee14 100644 --- a/ruff.toml +++ b/ruff.toml @@ -31,7 +31,7 @@ exclude = [ ] # Same as Black. -line-length = 119 +line-length = 120 # Allow unused variables when underscore-prefixed. dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" From 5c7c522c623e56efbc89e47adfa5b59ebf775754 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 00:54:29 +0800 Subject: [PATCH 44/49] uncomment fire in requirements.txt due to usage in the example --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 515a4d88b..f5ef63c58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ channels==4.0.0 # docx==0.2.4 #faiss==1.5.3 faiss_cpu==1.7.4 -# fire==0.4.0 +fire==0.4.0 typer # godot==0.1.1 # google_api_python_client==2.93.0 From 66c0bce60bfffb3727f27554ee0cbb5d0fac8817 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 00:58:56 +0800 Subject: [PATCH 45/49] add proper space --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c6e22989b..9eeacbccb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ COPY . /app/metagpt WORKDIR /app/metagpt RUN mkdir workspace &&\ pip install --no-cache-dir -r requirements.txt &&\ - pip install -e. + pip install -e . # Running with an infinite loop using the tail command CMD ["sh", "-c", "tail -f /dev/null"] From 77ec9b823f985fc0f30bccb5a71b2eec18b77f1d Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 00:59:23 +0800 Subject: [PATCH 46/49] remove duplicate string --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 2968dd34d..8c09eaf73 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,6 @@ workspace tmp build -workspace dist data geckodriver.log From b3750d5947894779fbaff392b242e722e57a05d6 Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 11:52:11 +0800 Subject: [PATCH 47/49] refine code for prepare document. remove useless logic --- metagpt/actions/prepare_documents.py | 29 ++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 8d3445ae4..3c0885954 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -12,28 +12,29 @@ from pathlib import Path from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import DEFAULT_WORKSPACE_ROOT, DOCS_FILE_REPO, REQUIREMENT_FILENAME +from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.schema import Document from metagpt.utils.file_repository import FileRepository from metagpt.utils.git_repository import GitRepository class PrepareDocuments(Action): - def __init__(self, name="", context=None, llm=None): - super().__init__(name, context, llm) + """PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt.""" + + def _init_repo(self): + """Initialize the Git environment.""" + path = CONFIG.project_path + if not path: + name = CONFIG.project_name or FileRepository.new_filename() + path = Path(CONFIG.workspace_path) / name + + if path.exists() and not CONFIG.inc: + shutil.rmtree(path) + CONFIG.git_repo = GitRepository(local_path=path, auto_init=True) async def run(self, with_messages, **kwargs): - if not CONFIG.git_repo: - # Create and initialize the workspace folder, initialize the Git environment. - project_name = CONFIG.project_name or FileRepository.new_filename() - workdir = CONFIG.project_path - if not workdir and CONFIG.workspace_path: - workdir = Path(CONFIG.workspace_path) / project_name - workdir = Path(workdir or DEFAULT_WORKSPACE_ROOT / project_name) - if not CONFIG.inc and workdir.exists(): - shutil.rmtree(workdir) - CONFIG.git_repo = GitRepository() - CONFIG.git_repo.open(local_path=workdir, auto_init=True) + """Create and initialize the workspace folder, initialize the Git environment.""" + self._init_repo() # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) From f365348f49815c85fe4ca163647e66ad56ccd73f Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 11:59:59 +0800 Subject: [PATCH 48/49] add .pylintrc --- docs/.pylintrc | 639 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 639 insertions(+) create mode 100644 docs/.pylintrc diff --git a/docs/.pylintrc b/docs/.pylintrc new file mode 100644 index 000000000..9e8488bc7 --- /dev/null +++ b/docs/.pylintrc @@ -0,0 +1,639 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist=pydantic + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +#ignore-patterns=^\.# +ignore-patterns=(.)*_test\.py,test_(.)*\.py + + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=120 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.9 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + v, + e, + d, + m, + df, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + expression-not-assigned, + pointless-statement + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io From 1ab0ae99a90c54b6c8d104684a5127f91710e04c Mon Sep 17 00:00:00 2001 From: geekan Date: Wed, 20 Dec 2023 12:48:57 +0800 Subject: [PATCH 49/49] refine sop --- metagpt/actions/write_prd_an.py | 21 ++++++++++++++------- metagpt/roles/product_manager.py | 4 ++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index edd94a463..8698c739f 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -26,8 +26,8 @@ PROGRAMMING_LANGUAGE = ActionNode( ORIGINAL_REQUIREMENTS = ActionNode( key="Original Requirements", expected_type=str, - instruction="Place the polished, complete original requirements here.", - example="The game should have a leaderboard and multiple difficulty levels.", + instruction="Place the original user's requirements here.", + example="Create a 2048 game", ) PROJECT_NAME = ActionNode( @@ -41,7 +41,7 @@ PRODUCT_GOALS = ActionNode( key="Product Goals", expected_type=list[str], instruction="Provide up to three clear, orthogonal product goals.", - example=["Create an engaging user experience", "Ensure high performance", "Provide customizable features"], + example=["Create an engaging user experience", "Improve accessibility, be responsive", "More beautiful UI"], ) USER_STORIES = ActionNode( @@ -49,8 +49,11 @@ USER_STORIES = ActionNode( expected_type=list[str], instruction="Provide up to 3 to 5 scenario-based user stories.", example=[ - "As a user, I want to be able to choose difficulty levels", + "As a player, I want to be able to choose difficulty levels", "As a player, I want to see my score after each game", + "As a player, I want to get restart button when I lose", + "As a player, I want to see beautiful UI that make me feel good", + "As a player, I want to play game via mobile phone", ], ) @@ -58,7 +61,11 @@ COMPETITIVE_ANALYSIS = ActionNode( key="Competitive Analysis", expected_type=list[str], instruction="Provide 5 to 7 competitive products.", - example=["Python Snake Game: Simple interface, lacks advanced features"], + example=[ + "2048 Game A: Simple interface, lacks responsive features", + "play2048.co: Beautiful and responsive UI with my best score shown", + "2048game.com: Responsive UI with my best score shown, but many ads", + ], ) COMPETITIVE_QUADRANT_CHART = ActionNode( @@ -86,7 +93,7 @@ REQUIREMENT_ANALYSIS = ActionNode( key="Requirement Analysis", expected_type=str, instruction="Provide a detailed analysis of the requirements.", - example="The product should be user-friendly.", + example="", ) REQUIREMENT_POOL = ActionNode( @@ -107,7 +114,7 @@ ANYTHING_UNCLEAR = ActionNode( key="Anything UNCLEAR", expected_type=str, instruction="Mention any aspects of the project that are unclear and try to clarify them.", - example="...", + example="", ) ISSUE_TYPE = ActionNode( diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 7858d2caa..61263cb50 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -28,8 +28,8 @@ class ProductManager(Role): self, name: str = "Alice", profile: str = "Product Manager", - goal: str = "efficiently create a successful product", - constraints: str = "use same language as user requirement", + goal: str = "efficiently create a successful product that meets market demands and user expectations", + constraints: str = "utilize the same language as the user requirements for seamless communication", ) -> None: """ Initializes the ProductManager role with given attributes.