From ebc4fe4b179acfe8c373afb8e2ee922e15fb06c6 Mon Sep 17 00:00:00 2001 From: better629 Date: Tue, 19 Dec 2023 14:22:52 +0800 Subject: [PATCH] update ser&deser after env_refactor --- metagpt/actions/action.py | 24 ++--- metagpt/actions/prepare_documents.py | 2 - metagpt/actions/write_code.py | 13 +-- metagpt/actions/write_code_review.py | 38 ++++---- metagpt/actions/write_prd.py | 18 ++-- metagpt/environment.py | 9 +- metagpt/memory/memory.py | 16 ++-- metagpt/roles/architect.py | 10 +-- metagpt/roles/engineer.py | 13 +-- metagpt/roles/product_manager.py | 3 +- metagpt/roles/project_manager.py | 2 - metagpt/roles/role.py | 129 +++++++++------------------ metagpt/schema.py | 63 +++++++------ metagpt/team.py | 9 +- metagpt/utils/utils.py | 3 +- 15 files changed, 152 insertions(+), 200 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index c941d44b6..a21f575ea 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -7,23 +7,21 @@ """ from __future__ import annotations -import re -from typing import Optional, Any from typing import Optional, Any -from tenacity import retry, stop_after_attempt, wait_random_exponential + from pydantic import BaseModel, Field +from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM -from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.logs import logger +from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess from metagpt.utils.common import OutputParser from metagpt.utils.utils import general_after_log from metagpt.utils.utils import import_class - action_subclass_registry = {} @@ -31,9 +29,10 @@ class Action(BaseModel): name: str = "" llm: BaseGPTAPI = Field(default_factory=LLM, exclude=True) context = "" - prefix = "" # aask*时会加上prefix，作为system_message + prefix = "" # aask*时会加上prefix，作为system_message profile = "" # FIXME: USELESS - desc = "" # for skill manager + desc = "" # for skill manager + nodes = [] # content: Optional[str] = None # instruct_content: Optional[str] = None @@ -42,7 +41,7 @@ class Action(BaseModel): class Config: arbitrary_types_allowed = True - + def __init__(self, **kwargs: Any): super().__init__(**kwargs) @@ -64,10 +63,11 @@ class Action(BaseModel): """Set prefix for later usage""" self.prefix = prefix self.profile = profile + return self def __str__(self): return self.__class__.__name__ - + def __repr__(self): return self.__str__() @@ -110,16 +110,16 @@ class Action(BaseModel): content = await self.llm.aask(prompt, system_msgs) logger.debug(f"llm raw output:\n{content}") output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) - + if format == "json": parsed_data = llm_output_postprecess(output=content, schema=output_class.schema(), req_key="[/CONTENT]") else: # using markdown parser parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) - + logger.debug(parsed_data) instruct_content = output_class(**parsed_data) return ActionOutput(content, instruct_content) - + async def run(self, *args, **kwargs): """Run action""" raise NotImplementedError("The run method should be implemented in a subclass.") diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 8d3445ae4..af38b7eae 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -19,8 +19,6 @@ from metagpt.utils.git_repository import GitRepository class PrepareDocuments(Action): - def __init__(self, name="", context=None, llm=None): - super().__init__(name, context, llm) async def run(self, with_messages, **kwargs): if not CONFIG.git_repo: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index bad9a0890..046f9f456 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -16,9 +16,10 @@ """ import json -from tenacity import retry, stop_after_attempt, wait_random_exponential -from typing import List, Optional, Any +from typing import Optional + from pydantic import Field +from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action from metagpt.config import CONFIG @@ -30,8 +31,8 @@ from metagpt.const import ( TEST_OUTPUTS_FILE_REPO, ) from metagpt.llm import LLM -from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.logs import logger +from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository @@ -89,7 +90,7 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc class WriteCode(Action): name: str = "WriteCode" - context: Optional[str] = None + context: Optional[Document] = None llm: BaseGPTAPI = Field(default_factory=LLM) @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) @@ -131,7 +132,9 @@ class WriteCode(Action): logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) if not coding_context.code_doc: - coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace) + # avoid root_path pydantic ValidationError if use WriteCode alone + root_path = CONFIG.src_workspace if CONFIG.src_workspace else "" + coding_context.code_doc = Document(filename=coding_context.filename, root_path=root_path) coding_context.code_doc.content = code return coding_context diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 636f3f12a..f4ab0adfe 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -7,21 +7,19 @@ @Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the WriteCode object, rather than passing them in when calling the run function. """ -from typing import List, Optional, Any -from pydantic import Field -from tenacity import retry, stop_after_attempt, wait_fixed -from typing import List, Optional, Any +from typing import Optional + from pydantic import Field from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions import WriteCode -from metagpt.llm import LLM from metagpt.actions.action import Action from metagpt.config import CONFIG +from metagpt.llm import LLM from metagpt.logs import logger -from metagpt.schema import CodingContext from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.schema import CodingContext from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -39,7 +37,6 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc ``` """ - EXAMPLE_AND_INSTRUCTION = """ {format_example} @@ -127,7 +124,7 @@ REWRITE_CODE_TEMPLATE = """ class WriteCodeReview(Action): name: str = "WriteCodeReview" - context: Optional[str] = None + context: Optional[CodingContext] = None llm: BaseGPTAPI = Field(default_factory=LLM) @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) @@ -147,9 +144,15 @@ class WriteCodeReview(Action): iterative_code = self.context.code_doc.content k = CONFIG.code_review_k_times or 1 for i in range(k): - format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename) - task_content = self.context.task_doc.content if self.context.task_doc else "" - code_context = await WriteCode.get_codes(self.context.task_doc, exclude=self.context.filename) + format_example = FORMAT_EXAMPLE.format( + filename=self.context.code_doc.filename + ) + task_content = ( + self.context.task_doc.content if self.context.task_doc else "" + ) + code_context = await WriteCode.get_codes( + self.context.task_doc, exclude=self.context.filename + ) context = "\n".join( [ "## System Design\n" + str(self.context.design_doc) + "\n", @@ -162,11 +165,16 @@ class WriteCodeReview(Action): code=iterative_code, filename=self.context.code_doc.filename, ) - cr_prompt = EXAMPLE_AND_INSTRUCTION.format(format_example=format_example, ) - logger.info( - f"Code review and rewrite {self.context.code_doc.filename}: {i+1}/{k} | {len(iterative_code)=}, {len(self.context.code_doc.content)=}" + cr_prompt = EXAMPLE_AND_INSTRUCTION.format( + format_example=format_example, + ) + logger.info( + f"Code review and rewrite {self.context.code_doc.filename}: {i + 1}/{k} | {len(iterative_code)=}, " + f"{len(self.context.code_doc.content)=}" + ) + result, rewrited_code = await self.write_code_review_and_rewrite( + context_prompt, cr_prompt, self.context.code_doc.filename ) - result, rewrited_code = await self.write_code_review_and_rewrite(context_prompt, cr_prompt, self.context.code_doc.filename) if "LBTM" in result: iterative_code = rewrited_code elif "LGTM" in result: diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 8510733ac..e76e91272 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -15,8 +15,9 @@ from __future__ import annotations import json from pathlib import Path -from typing import List, Optional, Any -from pydantic import BaseModel, Field +from typing import Optional + +from pydantic import Field from metagpt.actions import Action, ActionOutput from metagpt.actions.action_node import ActionNode @@ -26,9 +27,6 @@ from metagpt.actions.write_prd_an import ( WP_ISSUE_TYPE_NODE, WRITE_PRD_NODE, ) -from metagpt.llm import LLM -from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG from metagpt.const import ( BUGFIX_FILENAME, @@ -38,13 +36,14 @@ from metagpt.const import ( PRDS_FILE_REPO, REQUIREMENT_FILENAME, ) +from metagpt.llm import LLM from metagpt.logs import logger +from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.schema import BugFixContext, Document, Documents, Message from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.mermaid import mermaid_to_file - CONTEXT_TEMPLATE = """ ### Project Name {project_name} @@ -75,7 +74,7 @@ class WritePRD(Action): # related to the PRD. If they are related, rewrite the PRD. docs_file_repo = CONFIG.git_repo.new_file_repository(relative_path=DOCS_FILE_REPO) requirement_doc = await docs_file_repo.get(filename=REQUIREMENT_FILENAME) - if await self._is_bugfix(requirement_doc.content): + if requirement_doc and await self._is_bugfix(requirement_doc.content): await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content) await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="") bug_fix = BugFixContext(filename=BUGFIX_FILENAME) @@ -144,7 +143,8 @@ class WritePRD(Action): async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: if not prd_doc: - prd = await self._run_new_requirement(requirements=[requirement_doc.content], *args, **kwargs) + prd = await self._run_new_requirement(requirements=[requirement_doc.content if requirement_doc else ""], + *args, **kwargs) new_prd_doc = Document( root_path=PRDS_FILE_REPO, filename=FileRepository.new_filename() + ".json", @@ -166,7 +166,7 @@ class WritePRD(Action): if not quadrant_chart: return pathname = ( - CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") ) if not pathname.parent.exists(): pathname.parent.mkdir(parents=True, exist_ok=True) diff --git a/metagpt/environment.py b/metagpt/environment.py index 19c77a03d..4c8d7d5e5 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -12,14 +12,12 @@ functionality is to be consolidated into the `Environment` class. """ import asyncio -from typing import Iterable, Set from pathlib import Path +from typing import Iterable, Set from pydantic import BaseModel, Field from metagpt.logs import logger -from metagpt.roles import Role -from metagpt.memory import Memory from metagpt.roles.role import Role, role_subclass_registry from metagpt.schema import Message from metagpt.utils.common import is_subscribed @@ -29,7 +27,6 @@ from metagpt.utils.utils import read_json_file, write_json_file class Environment(BaseModel): """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - """ roles: dict[str, Role] = Field(default_factory=dict) @@ -63,12 +60,11 @@ class Environment(BaseModel): roles_info.append({ "role_class": role.__class__.__name__, "module_name": role.__module__, - "role_name": role.name + "role_name": role.name, }) role.serialize(stg_path=stg_path.joinpath(f"roles/{role.__class__.__name__}_{role.name}")) write_json_file(roles_path, roles_info) - self.memory.serialize(stg_path) history_path = stg_path.joinpath("history.json") write_json_file(history_path, {"content": self.history}) @@ -92,6 +88,7 @@ class Environment(BaseModel): "history": history }) environment.add_roles(roles) + return environment def add_role(self, role: Role): diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index b647198e3..fe70358c9 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -8,16 +8,14 @@ """ import copy from collections import defaultdict - -from typing import Iterable, Type, Union, Optional, Set from pathlib import Path +from typing import Iterable, Set + from pydantic import BaseModel, Field -import json from metagpt.schema import Message from metagpt.utils.common import any_to_str, any_to_str_set from metagpt.utils.utils import read_json_file, write_json_file -from metagpt.utils.utils import import_class class Memory(BaseModel): @@ -30,10 +28,7 @@ class Memory(BaseModel): index = kwargs.get("index", {}) new_index = defaultdict(list) for action_str, value in index.items(): - action_dict = json.loads(action_str) - action_class = import_class("Action", "metagpt.actions.action") - action_obj = action_class.deser_class(action_dict) - new_index[action_obj] = [Message(**item_dict) for item_dict in value] + new_index[action_str] = [Message(**item_dict) for item_dict in value] kwargs["index"] = new_index super(Memory, self).__init__(**kwargs) self.index = new_index @@ -43,9 +38,8 @@ class Memory(BaseModel): obj_dict = super(Memory, self).dict(*args, **kwargs) new_obj_dict = copy.deepcopy(obj_dict) new_obj_dict["index"] = {} - for action, value in obj_dict["index"].items(): - action_ser = json.dumps(action.ser_class()) - new_obj_dict["index"][action_ser] = value + for action_str, value in obj_dict["index"].items(): + new_obj_dict["index"][action_str] = value return new_obj_dict def serialize(self, stg_path: Path): diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 266ffc256..9edfe33d9 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -23,11 +23,11 @@ class Architect(Role): constraints (str): Constraints or guidelines for the architect. """ - name: str = "Bob" - profile: str = Field(default="Architect", alias='profile') - goal: str = "design a concise, usable, complete software system" - constraints: str = "make sure the architecture is simple enough and use appropriate open source libraries." \ - "Use same language as user requirement" + name: str = Field(default="Bob") + profile: str = Field(default="Architect") + goal: str = Field(default="design a concise, usable, complete software system") + constraints: str = Field(default="make sure the architecture is simple enough and use appropriate open source " + "libraries. Use same language as user requirement") def __init__(self, **kwargs) -> None: super().__init__(**kwargs) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index ad3d0f66a..206afb38c 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -18,12 +18,14 @@ """ from __future__ import annotations -from pydantic import Field + import json from collections import defaultdict from pathlib import Path from typing import Set +from pydantic import Field + from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.fix_bug import FixBug from metagpt.actions.summarize_code import SummarizeCode @@ -45,7 +47,6 @@ from metagpt.schema import ( ) from metagpt.utils.common import any_to_str, any_to_str_set - IS_PASS_PROMPT = """ {context} @@ -69,15 +70,15 @@ class Engineer(Role): use_code_review (bool): Whether to use code review. """ name: str = "Alex" - role_profile: str = Field(default="Engineer", alias='profile') + profile: str = Field(default="Engineer") goal: str = "write elegant, readable, extensible, efficient code" constraints: str = "the code should conform to standards like google-style and be modular and maintainable. " \ - "Use same language as user requirement", + "Use same language as user requirement" n_borg: int = 1 use_code_review: bool = False code_todos: list = [] summarize_todos = [] - + def __init__(self, **kwargs) -> None: super().__init__(**kwargs) @@ -211,7 +212,7 @@ class Engineer(Role): @staticmethod async def _new_coding_context( - filename, src_file_repo, task_file_repo, design_file_repo, dependency + filename, src_file_repo, task_file_repo, design_file_repo, dependency ) -> CodingContext: old_code_doc = await src_file_repo.get(filename) if not old_code_doc: diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 30017b60d..d054b94f5 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -26,13 +26,14 @@ class ProductManager(Role): constraints (str): Constraints or limitations for the project manager. """ name: str = "Alice" - role_profile: str = Field(default="Product Manager", alias='profile') + profile: str = Field(default="Product Manager") goal: str = "efficiently create a successful product" constraints: str = "use same language as user requiremen" """ Represents a Product Manager role responsible for product development and management. """ + def __init__(self, **kwargs) -> None: super().__init__(**kwargs) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index d885f2ee6..ec93e609b 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -24,8 +24,6 @@ class ProjectManager(Role): """ name: str = Field(default="Eve") profile: str = Field(default="Project Manager") - - goal: str = "reak down tasks according to PRD/technical design, generate a task list, and analyze task " \ "dependencies to start with the prerequisite modules" constraints: str = "use same language as user requirement" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index bed5a38e7..dbbaf8713 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -20,42 +20,26 @@ """ from __future__ import annotations + from enum import Enum -from typing import Iterable, Set, Type from pathlib import Path +from typing import Iterable, Set, Type, Any + from pydantic import BaseModel, Field from metagpt.actions.action import Action, ActionOutput, action_subclass_registry from metagpt.actions.action_node import ActionNode from metagpt.actions.add_requirement import UserRequirement - -from pathlib import Path - -from typing import ( - Iterable, - Type, - Any -) -from pydantic import BaseModel, Field, validator - -# from metagpt.environment import Environment -from metagpt.config import CONFIG -from metagpt.actions.action import Action, ActionOutput, action_subclass_registry +from metagpt.const import SERDESER_PATH from metagpt.llm import LLM -from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.logs import logger +from metagpt.memory import Memory +from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.provider.human_provider import HumanProvider from metagpt.schema import Message, MessageQueue from metagpt.utils.common import any_to_str from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output -from metagpt.memory import Memory -from metagpt.provider.human_provider import HumanProvider - from metagpt.utils.utils import read_json_file, write_json_file, import_class -from metagpt.provider.base_gpt_api import BaseGPTAPI - -from metagpt.utils.utils import read_json_file, write_json_file, import_class, role_raise_decorator -from metagpt.const import SERDESER_PATH - PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -65,12 +49,14 @@ Please note that only the text between the first and second "===" is information {history} === -You can now choose one of the following stages to decide the stage you need to go in the next step: +Your previous stage: {previous_state} + +Now choose one of the following stages you need to go to in the next step: {states} Just answer a number between 0-{n_states}, choose the most suitable stage according to the understanding of the conversation. Please note that the answer only needs a number, no need to add any other text. -If there is no conversation record, choose 0. +If you think you have completed your goal and don't need to go to any of the stages, return -1. Do not answer anything else, and do not add any other information in your answer. """ @@ -106,7 +92,7 @@ class RoleSetting(BaseModel): def __str__(self): return f"{self.name}({self.profile})" - + def __repr__(self): return self.__str__() @@ -115,37 +101,21 @@ class RoleContext(BaseModel): """Role Runtime Context""" # # env exclude=True to avoid `RecursionError: maximum recursion depth exceeded in comparison` env: "Environment" = Field(default=None, exclude=True) - msg_buffer: MessageQueue = Field(default_factory=MessageQueue) # Message Buffer with Asynchronous Updates + # TODO judge if ser&deser + msg_buffer: MessageQueue = Field(default_factory=MessageQueue, + exclude=True) # Message Buffer with Asynchronous Updates memory: Memory = Field(default_factory=Memory) # long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None, exclude=True) - watch: set[Type[Action]] = Field(default_factory=set) + watch: set[str] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[], exclude=True) # TODO not used - react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes + react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes max_react_loop: int = 1 - + class Config: arbitrary_types_allowed = True - def __init__(self, **kwargs): - watch_info = kwargs.get("watch", set()) - watch = set() - for item in watch_info: - action = Action.deser_class(item) - watch.update([action]) - kwargs["watch"] = watch - super(RoleContext, self).__init__(**kwargs) - - def dict(self, *args, **kwargs) -> "DictStrAny": - obj_dict = super(RoleContext, self).dict(*args, **kwargs) - watch = obj_dict.get("watch", set()) - watch_info = [] - for item in watch: - watch_info.append(item.ser_class()) - obj_dict["watch"] = watch_info - return obj_dict - def check(self, role_id: str): # if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: # self.long_term_memory.recover_memory(role_id, self) @@ -156,26 +126,16 @@ class RoleContext(BaseModel): def important_memory(self) -> list[Message]: """Get the information corresponding to the watched actions""" return self.memory.get_by_actions(self.watch) - + @property def history(self) -> list[Message]: return self.memory.get() -class _RoleInjector(type): - def __call__(cls, *args, **kwargs): - instance = super().__call__(*args, **kwargs) - - if not instance._rc.watch: - instance._watch([UserRequirement]) - - return instance - - role_subclass_registry = {} -class Role(BaseModel, metaclass=_RoleInjector): +class Role(BaseModel): """Role/Agent""" name: str = "" profile: str = "" @@ -189,7 +149,7 @@ class Role(BaseModel, metaclass=_RoleInjector): _states: list[str] = Field(default=[]) _actions: list[Action] = Field(default=[]) _rc: RoleContext = Field(default=RoleContext) - _subscription: tuple = set() + _subscription: tuple[str] = set() # builtin variables recovered: bool = False # to tag if a recovered role @@ -203,6 +163,8 @@ class Role(BaseModel, metaclass=_RoleInjector): "_rc": RoleContext() } + __hash__ = object.__hash__ # support Role as hashable type in `Environment.members` + class Config: arbitrary_types_allowed = True exclude = ["_llm"] @@ -240,6 +202,9 @@ class Role(BaseModel, metaclass=_RoleInjector): else: object.__setattr__(self, key, self._private_attributes[key]) + if not self._rc.watch: + self._watch([UserRequirement]) + # deserialize child classes dynamically for inherited `role` object.__setattr__(self, "builtin_class_name", self.__class__.__name__) self.__fields__["builtin_class_name"].default = self.__class__.__name__ @@ -303,7 +268,7 @@ class Role(BaseModel, metaclass=_RoleInjector): def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) role_subclass_registry[cls.__name__] = cls - + def _reset(self): object.__setattr__(self, "_states", []) object.__setattr__(self, "_actions", []) @@ -338,7 +303,7 @@ class Role(BaseModel, metaclass=_RoleInjector): role_class = import_class(class_name=role_class_str, module_name=module_name) role = role_class(**role_info) # initiate particular Role - role.set_recovered(True) # set True to make a tag + role.set_recovered(True) # set True to make a tag role_memory = Memory.deserialize(stg_path) role.set_memory(role_memory) @@ -362,7 +327,7 @@ class Role(BaseModel, metaclass=_RoleInjector): for idx, action in enumerate(actions): if not isinstance(action, Action): ## 默认初始化 - i = action(llm=self._llm) + i = action(name="", llm=self._llm) else: if self._setting.is_human and not isinstance(action.llm, HumanProvider): logger.warning( @@ -437,24 +402,10 @@ class Role(BaseModel, metaclass=_RoleInjector): if env: env.set_subscription(self, self._subscription) - @property - def profile(self): - """Get the role description (position)""" - return self._setting.profile - - @property - def name(self): - """Get virtual user name""" - return self._setting.name - @property def subscription(self) -> Set: """The labels for messages to be consumed by the Role object.""" return self._subscription - - def set_env(self, env: "Environment"): - """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" - self._rc.env = env def _get_prefix(self): """Get the role prefix""" @@ -466,7 +417,7 @@ class Role(BaseModel, metaclass=_RoleInjector): "goal": self.goal, "constraints": self.constraints }) - + async def _think(self) -> None: """Think about what to do and decide on the next action""" if len(self._actions) == 1: @@ -475,7 +426,7 @@ class Role(BaseModel, metaclass=_RoleInjector): return if self.recovered and self._rc.state >= 0: self._set_state(self._rc.state) # action to run from recovered state - self.recovered = False # avoid max_react_loop out of work + self.recovered = False # avoid max_react_loop out of work return prompt = self._get_prefix() @@ -498,7 +449,7 @@ class Role(BaseModel, metaclass=_RoleInjector): if next_state == -1: logger.info(f"End actions with {next_state=}") self._set_state(next_state) - + async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.important_memory) @@ -535,8 +486,8 @@ class Role(BaseModel, metaclass=_RoleInjector): if news_text: logger.debug(f"{self._setting} observed: {news_text}") return len(self._rc.news) - - def _publish_message(self, msg): + + def publish_message(self, msg): """If the role belongs to env, then the role's messages will be broadcast to env""" if not msg: return @@ -557,7 +508,7 @@ class Role(BaseModel, metaclass=_RoleInjector): Use llm to select actions in _think dynamically """ actions_taken = 0 - rsp = Message("No actions taken yet") # will be overwritten after Role _act + rsp = Message(content="No actions taken yet") # will be overwritten after Role _act while actions_taken < self._rc.max_react_loop: # think await self._think() @@ -580,7 +531,7 @@ class Role(BaseModel, metaclass=_RoleInjector): async def _plan_and_act(self) -> Message: """first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... Use llm to come up with the plan dynamically.""" # TODO: to be implemented - return Message("") + return Message(content="") async def react(self) -> Message: """Entry to one of three strategies by which Role reacts to the observed Message""" @@ -613,24 +564,24 @@ class Role(BaseModel, metaclass=_RoleInjector): def get_memories(self, k=0) -> list[Message]: """A wrapper to return the most recent k memories of this role, return all when k=0""" return self._rc.memory.get(k=k) - + async def run(self, with_message=None): """Observe, and think and act based on the results of the observation""" if with_message: msg = None if isinstance(with_message, str): - msg = Message(with_message) + msg = Message(content=with_message) elif isinstance(with_message, Message): msg = with_message elif isinstance(with_message, list): - msg = Message("\n".join(with_message)) + msg = Message(content="\n".join(with_message)) self.put_message(msg) if not await self._observe(): # If there is no new information, suspend and wait logger.debug(f"{self._setting}: no news. waiting.") return - + rsp = await self.react() # Reset the next action to be taken. diff --git a/metagpt/schema.py b/metagpt/schema.py index 962850547..690f64128 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -13,6 +13,8 @@ 3. Add `id` to `Message` according to Section 2.2.3.1.1 of RFC 135. """ +from __future__ import annotations + import asyncio import json import os.path @@ -20,14 +22,9 @@ import uuid from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError from pathlib import Path -from typing import Dict, List, Optional, Set, TypedDict -from pydantic import BaseModel, Field - -from dataclasses import dataclass, field -from typing import Type, TypedDict, Union, Optional +from typing import Dict, List, Set, TypedDict, Optional, Any from pydantic import BaseModel, Field -from pydantic.main import ModelMetaclass from metagpt.config import CONFIG from metagpt.const import ( @@ -39,15 +36,7 @@ from metagpt.const import ( TASK_FILE_REPO, ) from metagpt.logs import logger -from metagpt.utils.serialize import actionoutout_schema_to_mapping, actionoutput_mapping_to_str, \ - actionoutput_str_to_mapping -from metagpt.utils.utils import import_class - from metagpt.utils.common import any_to_str, any_to_str_set -# from metagpt.utils.serialize import actionoutout_schema_to_mapping -# from metagpt.actions.action_output import ActionOutput -# from metagpt.actions.action import Action - from metagpt.utils.serialize import actionoutout_schema_to_mapping, actionoutput_mapping_to_str, \ actionoutput_str_to_mapping from metagpt.utils.utils import import_class @@ -58,7 +47,6 @@ class RawMessage(TypedDict): role: str - class Document(BaseModel): """ Represents a document. @@ -68,7 +56,7 @@ class Document(BaseModel): filename: str = "" content: str = "" - def get_meta(self) -> "Document": + def get_meta(self) -> Document: """Get metadata of the document. :return: A new Document instance with the same root path and filename. @@ -120,7 +108,6 @@ class Message(BaseModel): def __init__(self, **kwargs): instruct_content = kwargs.get("instruct_content", None) - cause_by = kwargs.get("cause_by", None) if instruct_content and not isinstance(instruct_content, BaseModel): ic = instruct_content mapping = actionoutput_str_to_mapping(ic["mapping"]) @@ -129,9 +116,11 @@ class Message(BaseModel): ic_obj = actionoutput_class.create_model_class(class_name=ic["class"], mapping=mapping) ic_new = ic_obj(**ic["value"]) kwargs["instruct_content"] = ic_new - if cause_by and not isinstance(cause_by, ModelMetaclass): - action_class = import_class("Action", "metagpt.actions.action") - kwargs["cause_by"] = action_class.deser_class(cause_by) + + kwargs["id"] = uuid.uuid4().hex + kwargs["cause_by"] = any_to_str(kwargs.get("cause_by", "")) + kwargs["sent_from"] = any_to_str(kwargs.get("sent_from", "")) + kwargs["send_to"] = any_to_str_set(kwargs.get("send_to", {MESSAGE_ROUTE_TO_ALL})) super(Message, self).__init__(**kwargs) def __setattr__(self, key, val): @@ -156,9 +145,6 @@ class Message(BaseModel): mapping = actionoutput_mapping_to_str(mapping) obj_dict["instruct_content"] = {"class": schema["title"], "mapping": mapping, "value": ic.dict()} - cb = self.cause_by - if cb: - obj_dict["cause_by"] = cb.ser_class() return obj_dict def __str__(self): @@ -214,11 +200,24 @@ class AIMessage(Message): super().__init__(content=content, role="assistant") -class MessageQueue: +class MessageQueue(BaseModel): """Message queue which supports asynchronous updates.""" - def __init__(self): - self._queue = Queue() + _queue: Queue = Field(default_factory=Queue) + + _private_attributes = { + "_queue": Queue() + } + + class Config: + arbitrary_types_allowed = True + + def __init__(self, **kwargs: Any): + for key in self._private_attributes.keys(): + if key in kwargs: + object.__setattr__(self, key, kwargs[key]) + else: + object.__setattr__(self, key, self._private_attributes[key]) def pop(self) -> Message | None: """Pop one message from the queue.""" @@ -266,7 +265,7 @@ class MessageQueue: return json.dumps(lst) @staticmethod - def load(self, v) -> "MessageQueue": + def load(self, v) -> MessageQueue: """Convert the json string to the `MessageQueue` object.""" q = MessageQueue() try: @@ -287,7 +286,7 @@ class CodingContext(BaseModel): code_doc: Optional[Document] @staticmethod - def loads(val: str) -> "CodingContext" | None: + def loads(val: str) -> CodingContext | None: try: m = json.loads(val) return CodingContext(**m) @@ -301,7 +300,7 @@ class TestingContext(BaseModel): test_doc: Optional[Document] @staticmethod - def loads(val: str) -> "TestingContext" | None: + def loads(val: str) -> TestingContext | None: try: m = json.loads(val) return TestingContext(**m) @@ -322,7 +321,7 @@ class RunCodeContext(BaseModel): output: Optional[str] @staticmethod - def loads(val: str) -> "RunCodeContext" | None: + def loads(val: str) -> RunCodeContext | None: try: m = json.loads(val) return RunCodeContext(**m) @@ -336,7 +335,7 @@ class RunCodeResult(BaseModel): stderr: str @staticmethod - def loads(val: str) -> "RunCodeResult" | None: + def loads(val: str) -> RunCodeResult | None: try: m = json.loads(val) return RunCodeResult(**m) @@ -351,7 +350,7 @@ class CodeSummarizeContext(BaseModel): reason: str = "" @staticmethod - def loads(filenames: List) -> "CodeSummarizeContext": + def loads(filenames: List) -> CodeSummarizeContext: ctx = CodeSummarizeContext() for filename in filenames: if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO): diff --git a/metagpt/team.py b/metagpt/team.py index bd02508c4..30e3dc618 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -8,18 +8,19 @@ Section 2.2.3.3 of RFC 135. """ from pathlib import Path + from pydantic import BaseModel, Field from metagpt.actions import UserRequirement from metagpt.config import CONFIG from metagpt.const import MESSAGE_ROUTE_TO_ALL +from metagpt.const import SERDESER_PATH from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import NoMoneyException from metagpt.utils.utils import read_json_file, write_json_file, serialize_decorator -from metagpt.const import SERDESER_PATH class Team(BaseModel): @@ -39,9 +40,9 @@ class Team(BaseModel): stg_path = SERDESER_PATH.joinpath("team") if stg_path is None else stg_path team_info_path = stg_path.joinpath("team_info.json") - write_json_file(team_info_path, self.dict(exclude={"environment": True})) + write_json_file(team_info_path, self.dict(exclude={"env": True})) - self.environment.serialize(stg_path.joinpath("environment")) # save environment alone + self.env.serialize(stg_path.joinpath("environment")) # save environment alone @classmethod def recover(cls, stg_path: Path) -> "Team": @@ -60,7 +61,7 @@ class Team(BaseModel): # recover environment environment = Environment.deserialize(stg_path=stg_path.joinpath("environment")) - team_info.update({"environment": environment}) + team_info.update({"env": environment}) team = Team(**team_info) return team diff --git a/metagpt/utils/utils.py b/metagpt/utils/utils.py index 35df654d7..57da57b00 100644 --- a/metagpt/utils/utils.py +++ b/metagpt/utils/utils.py @@ -9,6 +9,7 @@ from pathlib import Path import importlib from tenacity import _utils import traceback +from pydantic.json import pydantic_encoder from metagpt.logs import logger @@ -46,7 +47,7 @@ def write_json_file(json_file: str, data: list, encoding=None): folder_path.mkdir(parents=True, exist_ok=True) with open(json_file, "w", encoding=encoding) as fout: - json.dump(data, fout, ensure_ascii=False, indent=4) + json.dump(data, fout, ensure_ascii=False, indent=4, default=pydantic_encoder) def import_class(class_name: str, module_name: str) -> type: