diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 535c25cb9..62434e7f8 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -8,13 +8,18 @@ from __future__ import annotations -from typing import Optional, Any +from typing import Any, Optional from pydantic import BaseModel, Field from metagpt.llm import LLM from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.schema import CodingContext, CodeSummarizeContext, TestingContext, RunCodeContext +from metagpt.schema import ( + CodeSummarizeContext, + CodingContext, + RunCodeContext, + TestingContext, +) action_subclass_registry = {} diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 839acdc2e..9dc6862f9 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -17,7 +17,7 @@ from metagpt.config import CONFIG from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO from metagpt.llm import LLM, BaseGPTAPI from metagpt.logs import logger -from metagpt.schema import RunCodeResult, RunCodeContext +from metagpt.schema import RunCodeContext, RunCodeResult from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index f5e122356..055365421 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -45,9 +45,11 @@ class WriteDesign(Action): name: str = "" context: Optional[str] = None llm: BaseGPTAPI = Field(default_factory=LLM) - desc: str = "Based on the PRD, think about the system design, and design the corresponding APIs, " \ - "data structures, library tables, processes, and paths. Please provide your design, feedback " \ - "clearly and in detail." + desc: str = ( + "Based on the PRD, think about the system design, and design the corresponding APIs, " + "data structures, library tables, processes, and paths. Please provide your design, feedback " + "clearly and in detail." + ) async def run(self, with_messages: Message, schema: str = CONFIG.prompt_schema): # Use `git diff` to identify which PRD documents have been modified in the `docs/prds` directory. diff --git a/metagpt/actions/fix_bug.py b/metagpt/actions/fix_bug.py index eea40c91a..56b488218 100644 --- a/metagpt/actions/fix_bug.py +++ b/metagpt/actions/fix_bug.py @@ -9,6 +9,7 @@ from metagpt.actions import Action class FixBug(Action): """Fix bug action without any implementation details""" + name: str = "FixBug" async def run(self, *args, **kwargs): diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 9b5128cbd..696dc9a89 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -25,6 +25,7 @@ from metagpt.utils.git_repository import GitRepository class PrepareDocuments(Action): """PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt.""" + name: str = "PrepareDocuments" context: Optional[str] = None llm: BaseGPTAPI = Field(default_factory=LLM) diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index ea16c8891..bca9b337d 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -24,7 +24,7 @@ from metagpt.actions.action import Action from metagpt.config import CONFIG from metagpt.llm import LLM, BaseGPTAPI from metagpt.logs import logger -from metagpt.schema import RunCodeResult, RunCodeContext +from metagpt.schema import RunCodeContext, RunCodeResult from metagpt.utils.exceptions import handle_exception PROMPT_TEMPLATE = """ diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 3f110c370..6ab7becb6 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -5,18 +5,18 @@ @Author : alexanderwu @File : search_google.py """ +from typing import Optional + import pydantic -from typing import Optional, Any -from pydantic import BaseModel, Field +from pydantic import Field, root_validator from metagpt.actions import Action +from metagpt.config import CONFIG, Config from metagpt.llm import LLM -from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.config import Config, CONFIG from metagpt.logs import logger +from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.schema import Message from metagpt.tools.search_engine import SearchEngine -from pydantic import root_validator SEARCH_AND_SUMMARIZE_SYSTEM = """### Requirements 1. Please summarize the latest dialogue based on the reference information (secondary) and dialogue history (primary). Do not include text that is irrelevant to the conversation. @@ -120,7 +120,7 @@ class SearchAndSummarize(Action): engine = values.get("engine") search_func = values.get("search_func") config = Config() - + if engine is None: engine = config.search_engine try: @@ -135,7 +135,7 @@ class SearchAndSummarize(Action): if self.search_engine is None: logger.warning("Configure one of SERPAPI_API_KEY, SERPER_API_KEY, GOOGLE_API_KEY to unlock full feature") return "" - + query = context[-1].content # logger.debug(query) rsp = await self.search_engine.run(query) @@ -144,9 +144,9 @@ class SearchAndSummarize(Action): logger.error("empty rsp...") return "" # logger.info(rsp) - + system_prompt = [system_text] - + prompt = SEARCH_AND_SUMMARIZE_PROMPT.format( ROLE=self.prefix, CONTEXT=rsp, diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 580069b74..1eba672a5 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -142,15 +142,9 @@ class WriteCodeReview(Action): iterative_code = self.context.code_doc.content k = CONFIG.code_review_k_times or 1 for i in range(k): - format_example = FORMAT_EXAMPLE.format( - filename=self.context.code_doc.filename - ) - task_content = ( - self.context.task_doc.content if self.context.task_doc else "" - ) - code_context = await WriteCode.get_codes( - self.context.task_doc, exclude=self.context.filename - ) + format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) + task_content = self.context.task_doc.content if self.context.task_doc else "" + code_context = await WriteCode.get_codes(self.context.task_doc, exclude=self.context.filename) context = "\n".join( [ "## System Design\n" + str(self.context.design_doc) + "\n", diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index df66e6442..1223e5486 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -143,8 +143,9 @@ class WritePRD(Action): async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: if not prd_doc: - prd = await self._run_new_requirement(requirements=[requirement_doc.content if requirement_doc else ""], - *args, **kwargs) + prd = await self._run_new_requirement( + requirements=[requirement_doc.content if requirement_doc else ""], *args, **kwargs + ) new_prd_doc = Document( root_path=PRDS_FILE_REPO, filename=FileRepository.new_filename() + ".json", diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index fa3931ba6..9eb0bdbb6 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -9,14 +9,15 @@ """ from typing import Optional + from pydantic import Field -from metagpt.llm import LLM -from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.actions.action import Action from metagpt.config import CONFIG from metagpt.const import TEST_CODES_FILE_REPO +from metagpt.llm import LLM from metagpt.logs import logger +from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.schema import Document, TestingContext from metagpt.utils.common import CodeParser diff --git a/metagpt/environment.py b/metagpt/environment.py index ab296557f..58569ec08 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -56,12 +56,14 @@ class Environment(BaseModel): roles_path = stg_path.joinpath("roles.json") roles_info = [] for role_key, role in self.roles.items(): - roles_info.append({ - "role_class": role.__class__.__name__, - "module_name": role.__module__, - "role_name": role.name, - "role_sub_tags": list(self.members.get(role)) - }) + roles_info.append( + { + "role_class": role.__class__.__name__, + "module_name": role.__module__, + "role_name": role.name, + "role_sub_tags": list(self.members.get(role)), + } + ) role.serialize(stg_path=stg_path.joinpath(f"roles/{role.__class__.__name__}_{role.name}")) write_json_file(roles_path, roles_info) @@ -70,7 +72,7 @@ class Environment(BaseModel): @classmethod def deserialize(cls, stg_path: Path) -> "Environment": - """ stg_path: ./storage/team/environment/ """ + """stg_path: ./storage/team/environment/""" roles_path = stg_path.joinpath("roles.json") roles_info = read_json_file(roles_path) roles = [] @@ -83,9 +85,7 @@ class Environment(BaseModel): history = read_json_file(stg_path.joinpath("history.json")) history = history.get("content") - environment = Environment(**{ - "history": history - }) + environment = Environment(**{"history": history}) environment.add_roles(roles) return environment diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 76a8deabb..710074f81 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -5,9 +5,7 @@ """ from typing import Optional -from pydantic import Field -from typing import Optional from pydantic import Field from metagpt.logs import logger @@ -22,6 +20,7 @@ class LongTermMemory(Memory): - recover memory when it staruped - update memory when it changed """ + memory_storage: MemoryStorage = Field(default_factory=MemoryStorage) rc: Optional["RoleContext"] = None msg_from_recover: bool = False diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 076db832a..e9891ed00 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -6,7 +6,6 @@ @File : memory.py @Modified By: mashenquan, 2023-11-1. According to RFC 116: Updated the type of index key. """ -import copy from collections import defaultdict from pathlib import Path from typing import Iterable, Set @@ -14,11 +13,17 @@ from typing import Iterable, Set from pydantic import BaseModel, Field from metagpt.schema import Message -from metagpt.utils.common import any_to_str, any_to_str_set, read_json_file, write_json_file +from metagpt.utils.common import ( + any_to_str, + any_to_str_set, + read_json_file, + write_json_file, +) class Memory(BaseModel): """The most basic memory: super-memory""" + storage: list[Message] = [] index: dict[str, list[Message]] = Field(default_factory=defaultdict(list)) @@ -32,14 +37,14 @@ class Memory(BaseModel): self.index = new_index def serialize(self, stg_path: Path): - """ stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/ """ + """stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/""" memory_path = stg_path.joinpath("memory.json") storage = self.dict() write_json_file(memory_path, storage) @classmethod def deserialize(cls, stg_path: Path) -> "Memory": - """ stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/""" + """stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/""" memory_path = stg_path.joinpath("memory.json") memory_dict = read_json_file(memory_path) @@ -68,7 +73,7 @@ class Memory(BaseModel): return [message for message in self.storage if content in message.content] def delete_newest(self) -> "Message": - """ delete the newest message from the storage""" + """delete the newest message from the storage""" if len(self.storage) > 0: newest_msg = self.storage.pop() if newest_msg.cause_by and newest_msg in self.index[newest_msg.cause_by]: diff --git a/metagpt/provider/postprecess/base_postprecess_plugin.py b/metagpt/provider/postprecess/base_postprecess_plugin.py index afcef2531..46646be91 100644 --- a/metagpt/provider/postprecess/base_postprecess_plugin.py +++ b/metagpt/provider/postprecess/base_postprecess_plugin.py @@ -4,7 +4,6 @@ from typing import Union -from metagpt.logs import logger from metagpt.utils.repair_llm_raw_output import ( RepairType, extract_content_from_output, diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index bd6cd110b..c6ceaccb7 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -5,7 +5,6 @@ @Author : alexanderwu @File : architect.py """ -from pydantic import Field from metagpt.actions import WritePRD from metagpt.actions.design_api import WriteDesign @@ -22,11 +21,14 @@ class Architect(Role): goal (str): Primary goal or responsibility of the architect. constraints (str): Constraints or guidelines for the architect. """ + name: str = "Bob" profile: str = "Architect" goal: str = "design a concise, usable, complete software system" - constraints: str = "make sure the architecture is simple enough and use appropriate open source " \ - "libraries. Use same language as user requirement" + constraints: str = ( + "make sure the architecture is simple enough and use appropriate open source " + "libraries. Use same language as user requirement" + ) def __init__(self, **kwargs) -> None: super().__init__(**kwargs) diff --git a/metagpt/roles/customer_service.py b/metagpt/roles/customer_service.py index b2033ac0b..777f62731 100644 --- a/metagpt/roles/customer_service.py +++ b/metagpt/roles/customer_service.py @@ -6,7 +6,6 @@ @File : sales.py """ from typing import Optional -from pydantic import Field from metagpt.roles import Sales @@ -27,14 +26,11 @@ DESC = """ class CustomerService(Sales): - name: str = "Xiaomei" profile: str = "Human customer service" desc: str = DESC store: Optional[str] = None - def __init__( - self, - **kwargs): + def __init__(self, **kwargs): super().__init__(**kwargs) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 337184068..e0234f378 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -24,8 +24,6 @@ from collections import defaultdict from pathlib import Path from typing import Set -from pydantic import Field - from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.fix_bug import FixBug from metagpt.actions.summarize_code import SummarizeCode @@ -69,11 +67,14 @@ class Engineer(Role): n_borg (int): Number of borgs. use_code_review (bool): Whether to use code review. """ + name: str = "Alex" profile: str = "Engineer" goal: str = "write elegant, readable, extensible, efficient code" - constraints: str = "the code should conform to standards like google-style and be modular and maintainable. " \ - "Use same language as user requirement" + constraints: str = ( + "the code should conform to standards like google-style and be modular and maintainable. " + "Use same language as user requirement" + ) n_borg: int = 1 use_code_review: bool = False code_todos: list = [] @@ -212,7 +213,7 @@ class Engineer(Role): @staticmethod async def _new_coding_context( - filename, src_file_repo, task_file_repo, design_file_repo, dependency + filename, src_file_repo, task_file_repo, design_file_repo, dependency ) -> CodingContext: old_code_doc = await src_file_repo.get(filename) if not old_code_doc: diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 6369688a5..c794ad2eb 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -7,7 +7,6 @@ @Modified By: mashenquan, 2023/11/27. Add `PrepareDocuments` action according to Section 2.2.3.5.1 of RFC 135. """ -from pydantic import Field from metagpt.actions import UserRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments @@ -25,6 +24,7 @@ class ProductManager(Role): goal (str): Goal of the product manager. constraints (str): Constraints or limitations for the product manager. """ + name: str = "Alice" profile: str = "Product Manager" goal: str = "efficiently create a successful product that meets market demands and user expectations" diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index bf572d1f8..1fad4afc2 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -5,7 +5,6 @@ @Author : alexanderwu @File : project_manager.py """ -from pydantic import Field from metagpt.actions import WriteTasks from metagpt.actions.design_api import WriteDesign @@ -22,10 +21,13 @@ class ProjectManager(Role): goal (str): Goal of the project manager. constraints (str): Constraints or limitations for the project manager. """ + name: str = "Eve" profile: str = "Project Manager" - goal: str = "break down tasks according to PRD/technical design, generate a task list, and analyze task " \ - "dependencies to start with the prerequisite modules" + goal: str = ( + "break down tasks according to PRD/technical design, generate a task list, and analyze task " + "dependencies to start with the prerequisite modules" + ) constraints: str = "use same language as user requirement" def __init__(self, **kwargs) -> None: diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 369e3dc63..5e509300b 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -15,13 +15,8 @@ of SummarizeCode. """ -from pydantic import Field -from metagpt.actions import ( - DebugError, - RunCode, - WriteTest, -) +from metagpt.actions import DebugError, RunCode, WriteTest from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( @@ -40,8 +35,9 @@ class QaEngineer(Role): name: str = "Edward" profile: str = "QaEngineer" goal: str = "Write comprehensive and robust tests to ensure codes will work as expected without bugs" - constraints: str = "The test code you write should conform to code standard like PEP8, be modular, " \ - "easy to read and maintain" + constraints: str = ( + "The test code you write should conform to code standard like PEP8, be modular, " "easy to read and maintain" + ) test_round_allowed: int = 5 def __init__(self, **kwargs): @@ -118,7 +114,8 @@ class QaEngineer(Role): ) run_code_context.code = None run_code_context.test_code = None - recipient = parse_recipient(result.summary) # the recipient might be Engineer or myself + # the recipient might be Engineer or myself + recipient = parse_recipient(result.summary) mappings = {"Engineer": "Alex", "QaEngineer": "Edward"} self.publish_message( Message( diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index f87c4e250..8c5743467 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -23,7 +23,7 @@ from __future__ import annotations from enum import Enum from pathlib import Path -from typing import Iterable, Set, Type, Any +from typing import Any, Iterable, Set, Type from pydantic import BaseModel, Field @@ -37,7 +37,13 @@ from metagpt.logs import logger from metagpt.memory import Memory from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.schema import Message, MessageQueue -from metagpt.utils.common import any_to_str, read_json_file, write_json_file, import_class, role_raise_decorator +from metagpt.utils.common import ( + any_to_str, + import_class, + read_json_file, + role_raise_decorator, + write_json_file, +) from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -82,18 +88,22 @@ class RoleReactMode(str, Enum): class RoleContext(BaseModel): """Role Runtime Context""" + # # env exclude=True to avoid `RecursionError: maximum recursion depth exceeded in comparison` env: "Environment" = Field(default=None, exclude=True) # TODO judge if ser&deser - msg_buffer: MessageQueue = Field(default_factory=MessageQueue, - exclude=True) # Message Buffer with Asynchronous Updates + msg_buffer: MessageQueue = Field( + default_factory=MessageQueue, exclude=True + ) # Message Buffer with Asynchronous Updates memory: Memory = Field(default_factory=Memory) # long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None, exclude=True) watch: set[str] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[], exclude=True) # TODO not used - react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes + react_mode: RoleReactMode = ( + RoleReactMode.REACT + ) # see `Role._set_react_mode` for definitions of the following two attributes max_react_loop: int = 1 class Config: @@ -120,6 +130,7 @@ role_subclass_registry = {} class Role(BaseModel): """Role/Agent""" + name: str = "" profile: str = "" goal: str = "" @@ -145,7 +156,7 @@ class Role(BaseModel): "_states": [], "_actions": [], "_rc": RoleContext(), - "_subscription": set() + "_subscription": set(), } __hash__ = object.__hash__ # support Role as hashable type in `Environment.members` @@ -206,14 +217,14 @@ class Role(BaseModel): return f"{self.name}({self.profile})" def serialize(self, stg_path: Path = None): - stg_path = SERDESER_PATH.joinpath(f"team/environment/roles/{self.__class__.__name__}_{self.name}") \ - if stg_path is None else stg_path + stg_path = ( + SERDESER_PATH.joinpath(f"team/environment/roles/{self.__class__.__name__}_{self.name}") + if stg_path is None + else stg_path + ) role_info = self.dict(exclude={"_rc": {"memory": True, "msg_buffer": True}, "_llm": True}) - role_info.update({ - "role_class": self.__class__.__name__, - "module_name": self.__module__ - }) + role_info.update({"role_class": self.__class__.__name__, "module_name": self.__module__}) role_info_path = stg_path.joinpath("role_info.json") write_json_file(role_info_path, role_info) @@ -221,7 +232,7 @@ class Role(BaseModel): @classmethod def deserialize(cls, stg_path: Path) -> "Role": - """ stg_path = ./storage/team/environment/roles/{role_class}_{role_name}""" + """stg_path = ./storage/team/environment/roles/{role_class}_{role_name}""" role_info_path = stg_path.joinpath("role_info.json") role_info = read_json_file(role_info_path) @@ -328,12 +339,9 @@ class Role(BaseModel): """Get the role prefix""" if self.desc: return self.desc - return PREFIX_TEMPLATE.format(**{ - "profile": self.profile, - "name": self.name, - "goal": self.goal, - "constraints": self.constraints - }) + return PREFIX_TEMPLATE.format( + **{"profile": self.profile, "name": self.name, "goal": self.goal, "constraints": self.constraints} + ) async def _think(self) -> None: """Think about what to do and decide on the next action""" diff --git a/metagpt/roles/sales.py b/metagpt/roles/sales.py index fd5a42915..ba0a6fc6b 100644 --- a/metagpt/roles/sales.py +++ b/metagpt/roles/sales.py @@ -7,7 +7,6 @@ """ from typing import Optional -from pydantic import Field from metagpt.actions import SearchAndSummarize from metagpt.roles import Role @@ -15,7 +14,6 @@ from metagpt.tools import SearchEngineType class Sales(Role): - name: str = "Xiaomei" profile: str = "Retail sales guide" desc: str = "I am a sales guide in retail. My name is Xiaomei. I will answer some customer questions next, and I " diff --git a/metagpt/roles/searcher.py b/metagpt/roles/searcher.py index a5c399f47..a2136064f 100644 --- a/metagpt/roles/searcher.py +++ b/metagpt/roles/searcher.py @@ -35,7 +35,7 @@ class Searcher(Role): goal: str = "Provide search services for users" constraints: str = "Answer is rich and complete" engine: SearchEngineType = SearchEngineType.SERPAPI_GOOGLE - + def __init__(self, **kwargs) -> None: """ Initializes the Searcher role with given attributes. diff --git a/metagpt/schema.py b/metagpt/schema.py index 5103a4f20..4a9df7fe2 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -23,7 +23,7 @@ from abc import ABC from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError from pathlib import Path -from typing import Dict, List, Optional, Set, Type, TypedDict, TypeVar, Any +from typing import Any, Dict, List, Optional, Set, Type, TypedDict, TypeVar from pydantic import BaseModel, Field @@ -38,9 +38,12 @@ from metagpt.const import ( ) from metagpt.logs import logger from metagpt.utils.common import any_to_str, any_to_str_set, import_class -from metagpt.utils.serialize import actionoutout_schema_to_mapping, actionoutput_mapping_to_str, \ - actionoutput_str_to_mapping from metagpt.utils.exceptions import handle_exception +from metagpt.utils.serialize import ( + actionoutout_schema_to_mapping, + actionoutput_mapping_to_str, + actionoutput_str_to_mapping, +) class RawMessage(TypedDict): @@ -119,8 +122,9 @@ class Message(BaseModel): kwargs["instruct_content"] = ic_new kwargs["id"] = kwargs.get("id", uuid.uuid4().hex) - kwargs["cause_by"] = any_to_str(kwargs.get("cause_by", - import_class("UserRequirement", "metagpt.actions.add_requirement"))) + kwargs["cause_by"] = any_to_str( + kwargs.get("cause_by", import_class("UserRequirement", "metagpt.actions.add_requirement")) + ) kwargs["sent_from"] = any_to_str(kwargs.get("sent_from", "")) kwargs["send_to"] = any_to_str_set(kwargs.get("send_to", {MESSAGE_ROUTE_TO_ALL})) super(Message, self).__init__(**kwargs) @@ -138,7 +142,7 @@ class Message(BaseModel): super().__setattr__(key, new_val) def dict(self, *args, **kwargs) -> "DictStrAny": - """ overwrite the `dict` to dump dynamic pydantic model""" + """overwrite the `dict` to dump dynamic pydantic model""" obj_dict = super(Message, self).dict(*args, **kwargs) ic = self.instruct_content if ic: @@ -208,9 +212,7 @@ class MessageQueue(BaseModel): _queue: Queue = Field(default_factory=Queue) - _private_attributes = { - "_queue": Queue() - } + _private_attributes = {"_queue": Queue()} class Config: arbitrary_types_allowed = True diff --git a/metagpt/startup.py b/metagpt/startup.py index 59e0cb199..767a19a9d 100644 --- a/metagpt/startup.py +++ b/metagpt/startup.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import asyncio +from pathlib import Path import typer -from pathlib import Path from metagpt.config import CONFIG @@ -32,7 +32,7 @@ def startup( help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating " "unlimited. This parameter is used for debugging the workflow.", ), - recover_path: str = typer.Option(default=None, help="recover the project from existing serialized storage") + recover_path: str = typer.Option(default=None, help="recover the project from existing serialized storage"), ): """Run a startup. Be a boss.""" from metagpt.roles import ( diff --git a/metagpt/team.py b/metagpt/team.py index 0c1efb812..8b92ed47a 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -8,20 +8,24 @@ Section 2.2.3.3 of RFC 135. """ -from pathlib import Path import warnings +from pathlib import Path from pydantic import BaseModel, Field from metagpt.actions import UserRequirement from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_ALL -from metagpt.const import SERDESER_PATH +from metagpt.const import MESSAGE_ROUTE_TO_ALL, SERDESER_PATH from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import NoMoneyException, read_json_file, write_json_file, serialize_decorator +from metagpt.utils.common import ( + NoMoneyException, + read_json_file, + serialize_decorator, + write_json_file, +) class Team(BaseModel): @@ -51,12 +55,14 @@ class Team(BaseModel): @classmethod def deserialize(cls, stg_path: Path) -> "Team": - """ stg_path = ./storage/team """ + """stg_path = ./storage/team""" # recover team_info team_info_path = stg_path.joinpath("team_info.json") if not team_info_path.exists(): - raise FileNotFoundError("recover storage meta file `team_info.json` not exist, " - "not to recover and please start a new project.") + raise FileNotFoundError( + "recover storage meta file `team_info.json` not exist, " + "not to recover and please start a new project." + ) team_info: dict = read_json_file(team_info_path) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index e123e8fd9..ea3316d66 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -22,8 +22,7 @@ import re import traceback import typing from pathlib import Path -from typing import Any -from typing import List, Tuple, Union, get_args, get_origin +from typing import Any, List, Tuple, Union, get_args, get_origin import aiofiles import loguru @@ -219,7 +218,7 @@ class OutputParser: if start_index != -1 and end_index != -1: # Extract the structure part - structure_text = text[start_index: end_index + 1] + structure_text = text[start_index : end_index + 1] try: # Attempt to convert the text to a Python data type using ast.literal_eval @@ -439,7 +438,7 @@ def read_json_file(json_file: str, encoding=None) -> list[Any]: with open(json_file, "r", encoding=encoding) as fin: try: data = json.load(fin) - except Exception as exp: + except Exception: raise ValueError(f"read json file: {json_file} failed") return data @@ -474,9 +473,9 @@ def serialize_decorator(func): try: result = await func(self, *args, **kwargs) return result - except KeyboardInterrupt as kbi: + except KeyboardInterrupt: logger.error(f"KeyboardInterrupt occurs, start to serialize the project, exp:\n{format_trackback_info()}") - except Exception as exp: + except Exception: logger.error(f"Exception occurs, start to serialize the project, exp:\n{format_trackback_info()}") self.serialize() # Team.serialize @@ -491,14 +490,18 @@ def role_raise_decorator(func): logger.error(f"KeyboardInterrupt: {kbi} occurs, start to serialize the project") if self.latest_observed_msg: self._rc.memory.delete(self.latest_observed_msg) - raise Exception(format_trackback_info(limit=None)) # raise again to make it captured outside - except Exception as exp: + # raise again to make it captured outside + raise Exception(format_trackback_info(limit=None)) + except Exception: if self.latest_observed_msg: - logger.warning("There is a exception in role's execution, in order to resume, " - "we delete the newest role communication message in the role's memory.") + logger.warning( + "There is a exception in role's execution, in order to resume, " + "we delete the newest role communication message in the role's memory." + ) # remove role newest observed msg to make it observed again self._rc.memory.delete(self.latest_observed_msg) - raise Exception(format_trackback_info(limit=None)) # raise again to make it captured outside + # raise again to make it captured outside + raise Exception(format_trackback_info(limit=None)) return wrapper diff --git a/tests/metagpt/serialize_deserialize/test_environment.py b/tests/metagpt/serialize_deserialize/test_environment.py index b741b9c4b..096c1dd68 100644 --- a/tests/metagpt/serialize_deserialize/test_environment.py +++ b/tests/metagpt/serialize_deserialize/test_environment.py @@ -11,7 +11,11 @@ from metagpt.environment import Environment from metagpt.roles.project_manager import ProjectManager from metagpt.schema import Message from metagpt.utils.common import any_to_str -from tests.metagpt.serialize_deserialize.test_serdeser_base import RoleC, ActionOK, serdeser_path +from tests.metagpt.serialize_deserialize.test_serdeser_base import ( + ActionOK, + RoleC, + serdeser_path, +) def test_env_serialize(): @@ -35,10 +39,7 @@ def test_environment_serdeser(): ic_obj = ActionNode.create_model_class("prd", out_mapping) message = Message( - content="prd", - instruct_content=ic_obj(**out_data), - role="product manager", - cause_by=any_to_str(UserRequirement) + content="prd", instruct_content=ic_obj(**out_data), role="product manager", cause_by=any_to_str(UserRequirement) ) environment = Environment() diff --git a/tests/metagpt/serialize_deserialize/test_memory.py b/tests/metagpt/serialize_deserialize/test_memory.py index 0d756518b..5a40f5c3b 100644 --- a/tests/metagpt/serialize_deserialize/test_memory.py +++ b/tests/metagpt/serialize_deserialize/test_memory.py @@ -14,17 +14,14 @@ from tests.metagpt.serialize_deserialize.test_serdeser_base import serdeser_path def test_memory_serdeser(): - msg1 = Message(role="Boss", - content="write a snake game", - cause_by=UserRequirement) + msg1 = Message(role="Boss", content="write a snake game", cause_by=UserRequirement) out_mapping = {"field2": (list[str], ...)} out_data = {"field2": ["field2 value1", "field2 value2"]} ic_obj = ActionNode.create_model_class("system_design", out_mapping) - msg2 = Message(role="Architect", - instruct_content=ic_obj(**out_data), - content="system design content", - cause_by=WriteDesign) + msg2 = Message( + role="Architect", instruct_content=ic_obj(**out_data), content="system design content", cause_by=WriteDesign + ) memory = Memory() memory.add_batch([msg1, msg2]) @@ -40,17 +37,14 @@ def test_memory_serdeser(): def test_memory_serdeser_save(): - msg1 = Message(role="User", - content="write a 2048 game", - cause_by=UserRequirement) + msg1 = Message(role="User", content="write a 2048 game", cause_by=UserRequirement) out_mapping = {"field1": (list[str], ...)} out_data = {"field1": ["field1 value1", "field1 value2"]} ic_obj = ActionNode.create_model_class("system_design", out_mapping) - msg2 = Message(role="Architect", - instruct_content=ic_obj(**out_data), - content="system design content", - cause_by=WriteDesign) + msg2 = Message( + role="Architect", instruct_content=ic_obj(**out_data), content="system design content", cause_by=WriteDesign + ) memory = Memory() memory.add_batch([msg1, msg2]) diff --git a/tests/metagpt/serialize_deserialize/test_role.py b/tests/metagpt/serialize_deserialize/test_role.py index 88c7f7d8b..72da8a6fc 100644 --- a/tests/metagpt/serialize_deserialize/test_role.py +++ b/tests/metagpt/serialize_deserialize/test_role.py @@ -16,7 +16,12 @@ from metagpt.roles.product_manager import ProductManager from metagpt.roles.role import Role from metagpt.schema import Message from metagpt.utils.common import format_trackback_info -from tests.metagpt.serialize_deserialize.test_serdeser_base import RoleA, RoleB, RoleC, serdeser_path +from tests.metagpt.serialize_deserialize.test_serdeser_base import ( + RoleA, + RoleB, + RoleC, + serdeser_path, +) def test_roles(): @@ -75,12 +80,10 @@ async def test_role_serdeser_interrupt(): role_c = RoleC() shutil.rmtree(SERDESER_PATH.joinpath("team"), ignore_errors=True) - stg_path = SERDESER_PATH.joinpath(f"team", "environment", "roles", "{role_c.__class__.__name__}_{role_c.name}") + stg_path = SERDESER_PATH.joinpath("team", "environment", "roles", f"{role_c.__class__.__name__}_{role_c.name}") try: - await role_c.run( - with_message=Message(content="demo", cause_by=UserRequirement) - ) - except Exception as exp: + await role_c.run(with_message=Message(content="demo", cause_by=UserRequirement)) + except Exception: logger.error(f"Exception in `role_a.run`, detail: {format_trackback_info()}") role_c.serialize(stg_path) @@ -90,6 +93,4 @@ async def test_role_serdeser_interrupt(): assert new_role_a._rc.state == 1 with pytest.raises(Exception): - await role_c.run( - with_message=Message(content="demo", cause_by=UserRequirement) - ) + await role_c.run(with_message=Message(content="demo", cause_by=UserRequirement)) diff --git a/tests/metagpt/serialize_deserialize/test_schema.py b/tests/metagpt/serialize_deserialize/test_schema.py index 72b7153a7..0358265a9 100644 --- a/tests/metagpt/serialize_deserialize/test_schema.py +++ b/tests/metagpt/serialize_deserialize/test_schema.py @@ -14,12 +14,7 @@ def test_message_serdeser(): out_data = {"field3": "field3 value3", "field4": ["field4 value1", "field4 value2"]} ic_obj = ActionNode.create_model_class("code", out_mapping) - message = Message( - content="code", - instruct_content=ic_obj(**out_data), - role="engineer", - cause_by=WriteCode - ) + message = Message(content="code", instruct_content=ic_obj(**out_data), role="engineer", cause_by=WriteCode) ser_data = message.dict() assert ser_data["cause_by"] == "metagpt.actions.write_code.WriteCode" assert ser_data["instruct_content"]["class"] == "code" @@ -31,14 +26,11 @@ def test_message_serdeser(): def test_message_without_postprocess(): - """ to explain `instruct_content` should be postprocessed """ + """to explain `instruct_content` should be postprocessed""" out_mapping = {"field1": (list[str], ...)} out_data = {"field1": ["field1 value1", "field1 value2"]} ic_obj = ActionNode.create_model_class("code", out_mapping) - message = MockMessage( - content="code", - instruct_content=ic_obj(**out_data) - ) + message = MockMessage(content="code", instruct_content=ic_obj(**out_data)) ser_data = message.dict() assert ser_data["instruct_content"] == {"field1": ["field1 value1", "field1 value2"]} diff --git a/tests/metagpt/serialize_deserialize/test_serdeser_base.py b/tests/metagpt/serialize_deserialize/test_serdeser_base.py index eac083cf9..a66813489 100644 --- a/tests/metagpt/serialize_deserialize/test_serdeser_base.py +++ b/tests/metagpt/serialize_deserialize/test_serdeser_base.py @@ -16,7 +16,8 @@ serdeser_path = Path(__file__).absolute().parent.joinpath("..", "..", "data", "s class MockMessage(BaseModel): - """ to test normal dict without postprocess """ + """to test normal dict without postprocess""" + content: str = "" instruct_content: BaseModel = Field(default=None) @@ -26,9 +27,7 @@ class ActionPass(Action): async def run(self, messages: list["Message"]) -> ActionOutput: await asyncio.sleep(5) # sleep to make other roles can watch the executed Message - output_mapping = { - "result": (str, ...) - } + output_mapping = {"result": (str, ...)} pass_class = ActionNode.create_model_class("pass", output_mapping) pass_output = ActionOutput("ActionPass run passed", pass_class(**{"result": "pass result"})) diff --git a/tests/metagpt/serialize_deserialize/test_team.py b/tests/metagpt/serialize_deserialize/test_team.py index db6001325..dc41fa4ed 100644 --- a/tests/metagpt/serialize_deserialize/test_team.py +++ b/tests/metagpt/serialize_deserialize/test_team.py @@ -8,10 +8,16 @@ import shutil import pytest from metagpt.const import SERDESER_PATH -from metagpt.roles import ProjectManager, ProductManager, Architect -from metagpt.team import Team from metagpt.logs import logger -from tests.metagpt.serialize_deserialize.test_serdeser_base import RoleA, RoleB, RoleC, serdeser_path, ActionOK +from metagpt.roles import Architect, ProductManager, ProjectManager +from metagpt.team import Team +from tests.metagpt.serialize_deserialize.test_serdeser_base import ( + ActionOK, + RoleA, + RoleB, + RoleC, + serdeser_path, +) def test_team_deserialize(): @@ -110,10 +116,8 @@ async def test_team_recover_multi_roles_save(): role_a = RoleA() role_b = RoleB() - assert role_a.subscription == {"tests.metagpt.serialize_deserialize.test_serdeser_base.RoleA", - "RoleA"} - assert role_b.subscription == {"tests.metagpt.serialize_deserialize.test_serdeser_base.RoleB", - "RoleB"} + assert role_a.subscription == {"tests.metagpt.serialize_deserialize.test_serdeser_base.RoleA", "RoleA"} + assert role_b.subscription == {"tests.metagpt.serialize_deserialize.test_serdeser_base.RoleB", "RoleB"} assert role_b._rc.watch == {"tests.metagpt.serialize_deserialize.test_serdeser_base.ActionPass"} company = Team() diff --git a/tests/metagpt/serialize_deserialize/test_write_code.py b/tests/metagpt/serialize_deserialize/test_write_code.py index 0114c48da..65b8f456a 100644 --- a/tests/metagpt/serialize_deserialize/test_write_code.py +++ b/tests/metagpt/serialize_deserialize/test_write_code.py @@ -19,8 +19,9 @@ def test_write_design_serialize(): @pytest.mark.asyncio async def test_write_code_deserialize(): - context = CodingContext(filename="test_code.py", - design_doc=Document(content="write add function to calculate two numbers")) + context = CodingContext( + filename="test_code.py", design_doc=Document(content="write add function to calculate two numbers") + ) doc = Document(content=context.json()) action = WriteCode(context=doc) serialized_data = action.dict() diff --git a/tests/metagpt/serialize_deserialize/test_write_code_review.py b/tests/metagpt/serialize_deserialize/test_write_code_review.py index a15b744db..01026590c 100644 --- a/tests/metagpt/serialize_deserialize/test_write_code_review.py +++ b/tests/metagpt/serialize_deserialize/test_write_code_review.py @@ -18,7 +18,7 @@ def div(a: int, b: int = 0): context = CodingContext( filename="test_op.py", design_doc=Document(content="divide two numbers"), - code_doc=Document(content=code_content) + code_doc=Document(content=code_content), ) action = WriteCodeReview(context=context) diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index ee322368e..56e2b4fc3 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -6,9 +6,10 @@ @File : test_environment.py """ -import pytest from pathlib import Path +import pytest + from metagpt.actions import UserRequirement from metagpt.environment import Environment from metagpt.logs import logger @@ -16,7 +17,6 @@ from metagpt.manager import Manager from metagpt.roles import Architect, ProductManager, Role from metagpt.schema import Message - serdeser_path = Path(__file__).absolute().parent.joinpath("../data/serdeser_storage") @@ -26,23 +26,16 @@ def env(): def test_add_role(env: Environment): - role = ProductManager(name="Alice", - profile="product manager", - goal="create a new product", - constraints="limited resources") + role = ProductManager( + name="Alice", profile="product manager", goal="create a new product", constraints="limited resources" + ) env.add_role(role) assert env.get_role(role.profile) == role def test_get_roles(env: Environment): - role1 = Role(name="Alice", - profile="product manager", - goal="create a new product", - constraints="limited resources") - role2 = Role(name="Bob", - profile="engineer", - goal="develop the new product", - constraints="short deadline") + role1 = Role(name="Alice", profile="product manager", goal="create a new product", constraints="limited resources") + role2 = Role(name="Bob", profile="engineer", goal="develop the new product", constraints="short deadline") env.add_role(role1) env.add_role(role2) roles = env.get_roles() @@ -51,14 +44,10 @@ def test_get_roles(env: Environment): @pytest.mark.asyncio async def test_publish_and_process_message(env: Environment): - product_manager = ProductManager(name="Alice", - profile="Product Manager", - goal="做AI Native产品", - constraints="资源有限") - architect = Architect(name="Bob", - profile="Architect", - goal="设计一个可用、高效、较低成本的系统,包括数据结构与接口", - constraints="资源有限,需要节省成本") + product_manager = ProductManager(name="Alice", profile="Product Manager", goal="做AI Native产品", constraints="资源有限") + architect = Architect( + name="Bob", profile="Architect", goal="设计一个可用、高效、较低成本的系统,包括数据结构与接口", constraints="资源有限,需要节省成本" + ) env.add_roles([product_manager, architect]) diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index ef706abfa..1742757e8 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -9,12 +9,13 @@ """ import json + import pytest from metagpt.actions import Action -from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage from metagpt.actions.action_node import ActionNode from metagpt.actions.write_code import WriteCode +from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage from metagpt.utils.common import any_to_str @@ -77,24 +78,13 @@ def test_message_serdeser(): out_data = {"field3": "field3 value3", "field4": ["field4 value1", "field4 value2"]} ic_obj = ActionNode.create_model_class("code", out_mapping) - message = Message( - content="code", - instruct_content=ic_obj(**out_data), - role="engineer", - cause_by=WriteCode - ) + message = Message(content="code", instruct_content=ic_obj(**out_data), role="engineer", cause_by=WriteCode) message_dict = message.dict() assert message_dict["cause_by"] == "metagpt.actions.write_code.WriteCode" assert message_dict["instruct_content"] == { "class": "code", - "mapping": { - "field3": "(, Ellipsis)", - "field4": "(list[str], Ellipsis)" - }, - "value": { - "field3": "field3 value3", - "field4": ["field4 value1", "field4 value2"] - } + "mapping": {"field3": "(, Ellipsis)", "field4": "(list[str], Ellipsis)"}, + "value": {"field3": "field3 value3", "field4": ["field4 value1", "field4 value2"]}, } new_message = Message(**message_dict) diff --git a/tests/metagpt/test_team.py b/tests/metagpt/test_team.py index efd035bb2..930306b5e 100644 --- a/tests/metagpt/test_team.py +++ b/tests/metagpt/test_team.py @@ -2,8 +2,8 @@ # -*- coding: utf-8 -*- # @Desc : unittest of team -from metagpt.team import Team from metagpt.roles.project_manager import ProjectManager +from metagpt.team import Team def test_team():