diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 7bb5a151b..aefe6d39d 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -17,6 +17,7 @@ from metagpt.llm import LLM from metagpt.logs import logger from metagpt.utils.common import OutputParser from metagpt.utils.custom_decoder import CustomDecoder +from metagpt.utils.utils import import_class class Action(BaseModel): @@ -42,7 +43,37 @@ class Action(BaseModel): def __repr__(self): return self.__str__() - + + def serialize(self): + return { + "action_class": self.__class__.__name__, + "module_name": self.__module__, + "name": self.name + } + + @classmethod + def deserialize(cls, action_dict: dict): + action_class_str = action_dict.pop("action_class") + module_name = action_dict.pop("module_name") + action_class = import_class(action_class_str, module_name) + return action_class(**action_dict) + + @classmethod + def ser_class(cls): + """ serialize class type""" + return { + "action_class": cls.__name__, + "module_name": cls.__module__ + } + + @classmethod + def deser_class(cls, action_dict: dict): + """ deserialize class type """ + action_class_str = action_dict.pop("action_class") + module_name = action_dict.pop("module_name") + action_class = import_class(action_class_str, module_name) + return action_class + async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> str: """Append default prefix""" if not system_msgs: diff --git a/metagpt/const.py b/metagpt/const.py index 407ce803a..711546d03 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -42,6 +42,7 @@ TMP = PROJECT_ROOT / "tmp" RESEARCH_PATH = DATA_PATH / "research" TUTORIAL_PATH = DATA_PATH / "tutorial_docx" INVOICE_OCR_TABLE_PATH = DATA_PATH / "invoice_table" +SERDES_PATH = WORKSPACE_ROOT / "storage" # TODO to store `storage` under the individual generated project SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills" diff --git a/metagpt/environment.py b/metagpt/environment.py index 88ff145e0..e867ad6fc 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -7,12 +7,14 @@ """ import asyncio from typing import Iterable +from pathlib import Path from pydantic import BaseModel, Field from metagpt.memory import Memory from metagpt.roles import Role from metagpt.schema import Message +from metagpt.utils.utils import read_json_file, write_json_file class Environment(BaseModel): @@ -28,6 +30,42 @@ class Environment(BaseModel): class Config: arbitrary_types_allowed = True + def serialize(self, stg_path: Path): + roles_path = stg_path.joinpath("roles.json") + roles_info = [] + for role_key, role in self.roles.items(): + roles_info.append({ + "role_class": role.__class__.__name__, + "module_name": role.__module__, + "role_name": role.name + }) + role.serialize(stg_path=stg_path.joinpath(f"roles/{role.__class__.__name__}_{role.name}")) + write_json_file(roles_path, roles_info) + + self.memory.serialize(stg_path) + history_path = stg_path.joinpath("history.json") + write_json_file(history_path, {"content": self.history}) + + def deserialize(self, stg_path: Path): + """ stg_path: ./storage/team/environment/ """ + roles_path = stg_path.joinpath("roles.json") + roles_info = read_json_file(roles_path) + for role_info in roles_info: + role_class = role_info.get("role_class") + role_name = role_info.get("role_name") + + role_path = stg_path.joinpath(f"roles/{role_class}_{role_name}") + role = Role.deserialize(role_path) + + self.add_role(role) + + memory = Memory.deserialize(stg_path) + self.memory = memory + + history_path = stg_path.joinpath("history.json") + history = read_json_file(history_path) + self.history = history.get("content") + def add_role(self, role: Role): """增加一个在当前环境的角色, 默认为profile/role_profile Add a role in the current environment diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index c818fa707..a839bb038 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -7,9 +7,12 @@ """ from collections import defaultdict from typing import Iterable, Type +from pathlib import Path from metagpt.actions import Action from metagpt.schema import Message +from metagpt.utils.utils import read_json_file, write_json_file +from metagpt.utils.serialize import serialize_general_message, deserialize_general_message class Memory: @@ -20,6 +23,33 @@ class Memory: self.storage: list[Message] = [] self.index: dict[Type[Action], list[Message]] = defaultdict(list) + def serialize(self, stg_path: Path): + """ stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/ """ + memory_path = stg_path.joinpath("memory.json") + + storage = [] + for message in self.storage: + # msg_dict = message.serialize() + msg_dict = serialize_general_message(message) + storage.append(msg_dict) + + write_json_file(memory_path, storage) + + @classmethod + def deserialize(cls, stg_path: Path) -> "Memory": + """ stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/""" + memory_path = stg_path.joinpath("memory.json") + + memory = Memory() + memory_list = read_json_file(memory_path) + for message in memory_list: + # distinguish instruct_content type in message + # msg = Message.deserialize(message) + msg = deserialize_general_message(message) + memory.add(msg) + + return memory + def add(self, message: Message): """Add a new message to storage, while updating the index""" if message in self.storage: diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 9aae64188..eb5539f43 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -6,9 +6,11 @@ @File : role.py """ +import sys +from enum import Enum +import importlib from __future__ import annotations -import sys from types import SimpleNamespace from typing import ( Dict, @@ -28,6 +30,7 @@ from metagpt.llm import LLM from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory from metagpt.schema import Message +from metagpt.utils.utils import read_json_file, write_json_file, import_class PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -37,12 +40,14 @@ Please note that only the text between the first and second "===" is information {history} === -You can now choose one of the following stages to decide the stage you need to go in the next step: +Your previous stage: {previous_state} + +Now choose one of the following stages you need to go to in the next step: {states} Just answer a number between 0-{n_states}, choose the most suitable stage according to the understanding of the conversation. Please note that the answer only needs a number, no need to add any other text. -If there is no conversation record, choose 0. +If you think you have completed your goal and don't need to go to any of the stages, return -1. Do not answer anything else, and do not add any other information in your answer. """ @@ -56,6 +61,15 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi {name}: {result} """ +class RoleReactMode(str, Enum): + REACT = "react" + BY_ORDER = "by_order" + PLAN_AND_ACT = "plan_and_act" + + @classmethod + def values(cls): + return [item.value for item in cls] + class RoleSetting(BaseModel): """Role Settings""" @@ -81,6 +95,8 @@ class RoleContext(BaseModel): todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) + react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes + max_react_loop: int = 1 class Config: arbitrary_types_allowed = True @@ -102,6 +118,7 @@ class RoleContext(BaseModel): class Role(BaseModel): """Role/Agent""" + name: str = "" profile: str = "" goal: str = "" @@ -116,8 +133,8 @@ class Role(BaseModel): _rc: RoleContext = RoleContext() _private_attributes = { - '_setting': _setting, - '_role_id': _role_id, + "_setting': _setting, + "_role_id': _role_id, '_states': [], '_actions': [], '_actions_type': [] # 用于记录和序列化 @@ -144,9 +161,7 @@ class Role(BaseModel): def _reset(self): object.__setattr__(self, '_states', []) object.__setattr__(self, '_actions', []) - - @staticmethod def _process_class(class_str, module_name): cleaned_string = re.sub(r"[<>']", "", class_str).replace("class ", "") @@ -157,37 +172,167 @@ class Role(BaseModel): module_file = import_module(file_name, package=package_name) module = getattr(module_file, module_name) return module - + + def serialize(self, stg_path: Path): + role_info_path = stg_path.joinpath("role_info.json") + role_info = { + "role_class": self.__class__.__name__, + "module_name": self.__module__ + } + setting = self._setting.dict() + setting.pop("desc") + setting.pop("is_human") # not all inherited roles have this atrr + role_info.update(setting) + write_json_file(role_info_path, role_info) + + actions_info_path = stg_path.joinpath("actions/actions_info.json") + actions_info = [] + for action in self._actions: + actions_info.append(action.serialize()) + write_json_file(actions_info_path, actions_info) + + watches_info_path = stg_path.joinpath("watches/watches_info.json") + watches_info = [] + for watch in self._rc.watch: + watches_info.append(watch.ser_class()) + write_json_file(watches_info_path, watches_info) + + actions_todo_path = stg_path.joinpath("actions/todo.json") + actions_todo = { + "cur_state": self._rc.state, + "react_mode": self._rc.react_mode.value, + "max_react_loop": self._rc.max_react_loop + } + write_json_file(actions_todo_path, actions_todo) + + self._rc.memory.serialize(stg_path) + + @classmethod + def deserialize(cls, stg_path: Path) -> "Role": + """ stg_path = ./storage/team/environment/roles/{role_class}_{role_name}""" + role_info_path = stg_path.joinpath("role_info.json") + role_info = read_json_file(role_info_path) + + role_class_str = role_info.pop("role_class") + module_name = role_info.pop("module_name") + role_class = import_class(class_name=role_class_str, module_name=module_name) + + role = role_class(**role_info) # initiate particular Role + actions_info_path = stg_path.joinpath("actions/actions_info.json") + actions = [] + actions_info = read_json_file(actions_info_path) + for action_info in actions_info: + action = Action.deserialize(action_info) + actions.append(action) + + watches_info_path = stg_path.joinpath("watches/watches_info.json") + watches = [] + watches_info = read_json_file(watches_info_path) + for watch_info in watches_info: + action = Action.deser_class(watch_info) + watches.append(action) + + role.init_actions(actions) + role.watch(watches) + + actions_todo_path = stg_path.joinpath("actions/todo.json") + # recover self._rc.state + actions_todo = read_json_file(actions_todo_path) + max_react_loop = actions_todo.get("max_react_loop", 1) + cur_state = actions_todo.get("cur_state", -1) + role.set_state(cur_state) + role.set_recovered(True) + react_mode_str = actions_todo.get("react_mode", RoleReactMode.REACT.value) + if react_mode_str not in RoleReactMode.values(): + logger.warning(f"ReactMode: {react_mode_str} not in {RoleReactMode.values()}, use react as default") + react_mode_str = RoleReactMode.REACT.value + role.set_react_mode(RoleReactMode(react_mode_str), max_react_loop) + + role_memory = Memory.deserialize(stg_path) + role.set_memory(role_memory) + + return role + + def _reset(self): + self._states = [] + self._actions = [] + + def set_recovered(self, recovered: bool = False): + self._recovered = recovered + + def set_memory(self, memory: Memory): + self._rc.memory = memory + + def init_actions(self, actions): + self._init_actions(actions) + def _init_actions(self, actions): self._reset() for idx, action in enumerate(actions): if not isinstance(action, Action): ## 默认初始化 - i = action() + i = action("", llm=self._llm) else: + if self._setting.is_human and not isinstance(action.llm, HumanProvider): + logger.warning(f"is_human attribute does not take effect," + f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") i = action i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) self._states.append(f"{idx}. {action}") action_title = action.schema()["title"] self._actions_type.append(action_title) - + + def set_react_mode(self, react_mode: RoleReactMode, max_react_loop: int = 1): + self._set_react_mode(react_mode, max_react_loop) + + def _set_react_mode(self, react_mode: str, max_react_loop: int = 1): + """Set strategy of the Role reacting to observed Message. Variation lies in how + this Role elects action to perform during the _think stage, especially if it is capable of multiple Actions. + + Args: + react_mode (str): Mode for choosing action during the _think stage, can be one of: + "react": standard think-act loop in the ReAct paper, alternating thinking and acting to solve the task, i.e. _think -> _act -> _think -> _act -> ... + Use llm to select actions in _think dynamically; + "by_order": switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...; + "plan_and_act": first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... + Use llm to come up with the plan dynamically. + Defaults to "react". + max_react_loop (int): Maximum react cycles to execute, used to prevent the agent from reacting forever. + Take effect only when react_mode is react, in which we use llm to choose actions, including termination. + Defaults to 1, i.e. _think -> _act (-> return result and end) + """ + assert react_mode in RoleReactMode.values(), f"react_mode must be one of {RoleReactMode.values()}" + self._rc.react_mode = react_mode + if react_mode == RoleReactMode.REACT: + self._rc.max_react_loop = max_react_loop + + def watch(self, actions: Iterable[Type[Action]]): + self._watch(actions) + def _watch(self, actions: Iterable[Type[Action]]): """Listen to the corresponding behaviors""" self._rc.watch.update(actions) # check RoleContext after adding watch actions self._rc.check(self._role_id) - - def _set_state(self, state): + + def set_state(self, state: int): + self._set_state(state) + + def _set_state(self, state: int): """Update the current state.""" self._rc.state = state logger.debug(self._actions) - self._rc.todo = self._actions[self._rc.state] + self._rc.todo = self._actions[self._rc.state] if state >= 0 else None def set_env(self, env: 'Environment'): """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" self._rc.env = env + @property + def name(self): + return self._setting.name + @property def profile(self): """Get the role description (position)""" @@ -205,15 +350,25 @@ class Role(BaseModel): # If there is only one action, then only this one can be performed self._set_state(0) return + if self._recovered and self._rc.state >= 0: + self._set_state(self._rc.state) # action to run from recovered state + self._recovered = False # avoid max_react_loop out of work + return + prompt = self._get_prefix() prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1) + n_states=len(self._states) - 1, previous_state=self._rc.state) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") - if not next_state.isdigit() or int(next_state) not in range(len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}') - next_state = "0" - self._set_state(int(next_state)) + if (not next_state.isdigit() and next_state != "-1") \ + or int(next_state) not in range(-1, len(self._states)): + logger.warning(f'Invalid answer of state, {next_state=}, will be set to -1') + next_state = -1 + else: + next_state = int(next_state) + if next_state == -1: + logger.info(f"End actions with {next_state=}") + self._set_state(next_state) async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") @@ -237,8 +392,7 @@ class Role(BaseModel): observed = self._rc.env.memory.get_by_actions(self._rc.watch) - self._rc.news = self._rc.memory.find_news( - observed) # find news (previously unseen messages) from observed messages + self._rc.news = self._rc.memory.find_news(observed) # find news (previously unseen messages) from observed messages for i in env_msgs: self.recv(i) @@ -256,11 +410,47 @@ class Role(BaseModel): self._rc.env.publish_message(msg) async def _react(self) -> Message: - """Think first, then act""" - await self._think() - logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") - return await self._act() - + """Think first, then act, until the Role _think it is time to stop and requires no more todo. + This is the standard think-act loop in the ReAct paper, which alternates thinking and acting in task solving, i.e. _think -> _act -> _think -> _act -> ... + Use llm to select actions in _think dynamically + """ + actions_taken = 0 + rsp = Message("No actions taken yet") # will be overwritten after Role _act + while actions_taken < self._rc.max_react_loop: + # think + await self._think() + if self._rc.todo is None: + break + # act + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + rsp = await self._act() + actions_taken += 1 + return rsp # return output from the last action + + async def _act_by_order(self) -> Message: + """switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...""" + start_idx = self._rc.state if self._rc.state >= 0 else 0 # action to run from recovered state + for i in range(start_idx, len(self._states)): + self._set_state(i) + rsp = await self._act() + return rsp # return output from the last action + + async def _plan_and_act(self) -> Message: + """first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... Use llm to come up with the plan dynamically.""" + # TODO: to be implemented + return Message("") + + async def react(self) -> Message: + """Entry to one of three strategies by which Role reacts to the observed Message""" + if self._rc.react_mode == RoleReactMode.REACT: + rsp = await self._react() + elif self._rc.react_mode == RoleReactMode.BY_ORDER: + rsp = await self._act_by_order() + elif self._rc.react_mode == RoleReactMode.PLAN_AND_ACT: + rsp = await self._plan_and_act() + self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None + return rsp + def recv(self, message: Message) -> None: """add message to history.""" # self._history += f"\n{message}" @@ -276,6 +466,10 @@ class Role(BaseModel): return await self._react() + def get_memories(self, k=0) -> list[Message]: + """A wrapper to return the most recent k memories of this role, return all when k=0""" + return self._rc.memory.get(k=k) + async def run(self, message=None): """Observe, and think and act based on the results of the observation""" if message: @@ -290,7 +484,7 @@ class Role(BaseModel): logger.debug(f"{self._setting}: no news. waiting.") return - rsp = await self._react() + rsp = await self.react() # Publish the reply to the environment, waiting for the next subscriber to process self._publish_message(rsp) return rsp diff --git a/metagpt/schema.py b/metagpt/schema.py index bdca093c2..3374a7241 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -9,10 +9,14 @@ from __future__ import annotations from dataclasses import dataclass, field from typing import Type, TypedDict +import copy from pydantic import BaseModel from metagpt.logs import logger +# from metagpt.utils.serialize import actionoutout_schema_to_mapping +# from metagpt.actions.action_output import ActionOutput +# from metagpt.actions.action import Action class RawMessage(TypedDict): @@ -38,6 +42,46 @@ class Message: def __repr__(self): return self.__str__() + # def serialize(self): + # message_cp: Message = copy.deepcopy(self) + # ic = message_cp.instruct_content + # if ic: + # # model create by pydantic create_model like `pydantic.main.prd`, can't pickle.dump directly + # schema = ic.schema() + # mapping = actionoutout_schema_to_mapping(schema) + # + # message_cp.instruct_content = {"class": schema["title"], "mapping": mapping, "value": ic.dict()} + # cb = message_cp.cause_by + # if cb: + # message_cp.cause_by = cb.serialize() + # + # return message_cp.dict() + # + # @classmethod + # def deserialize(cls, message_dict: dict): + # instruct_content = message_dict.get("instruct_content") + # if instruct_content: + # ic = instruct_content + # ic_obj = ActionOutput.create_model_class(class_name=ic["class"], mapping=ic["mapping"]) + # ic_new = ic_obj(**ic["value"]) + # message_dict.instruct_content = ic_new + # cause_by = message_dict.get("cause_by") + # if cause_by: + # message_dict.cause_by = Action.deserialize(cause_by) + # + # return Message(**message_dict) + + def dict(self): + return { + "content": self.content, + "instruct_content": self.instruct_content, + "role": self.role, + "cause_by": self.cause_by, + "sent_from": self.sent_from, + "send_to": self.send_to, + "restricted_to": self.restricted_to + } + def to_dict(self) -> dict: return { "role": self.role, diff --git a/metagpt/team.py b/metagpt/team.py index 67d3ecec8..3b76e5ff4 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : software_company.py """ +from pathlib import Path from pydantic import BaseModel, Field from metagpt.actions import BossRequirement @@ -14,6 +15,7 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import NoMoneyException +from metagpt.utils.utils import read_json_file, write_json_file class Team(BaseModel): @@ -28,6 +30,30 @@ class Team(BaseModel): class Config: arbitrary_types_allowed = True + def serialize(self, stg_path: Path): + team_info_path = stg_path.joinpath("team_info.json") + write_json_file(team_info_path, { + "idea": self.idea, + "investment": self.investment + }) + + self.environment.serialize(stg_path.joinpath("environment")) + + def deserialize(self, stg_path: Path): + """ stg_path = ./storage/team """ + # recover team_info + team_info_path = stg_path.joinpath("team_info.json") + if not team_info_path.exists(): + logger.error("recover storage not exist, not to recover and continue run the old project.") + team_info = read_json_file(team_info_path) + self.investment = team_info.get("investment", 10.0) + self.idea = team_info.get("idea", "") + + # recover environment + environment_path = stg_path.joinpath("environment") + self.environment = Environment() + self.environment.deserialize(stg_path=environment_path) + def hire(self, roles: list[Role]): """Hire roles to cooperate""" self.environment.add_roles(roles) diff --git a/metagpt/utils/serialize.py b/metagpt/utils/serialize.py index 124176fcb..56a866f2e 100644 --- a/metagpt/utils/serialize.py +++ b/metagpt/utils/serialize.py @@ -4,13 +4,13 @@ import copy import pickle -from typing import Dict, List from metagpt.actions.action_output import ActionOutput from metagpt.schema import Message +from metagpt.actions.action import Action -def actionoutout_schema_to_mapping(schema: Dict) -> Dict: +def actionoutout_schema_to_mapping(schema: dict) -> dict: """ directly traverse the `properties` in the first level. schema structure likes @@ -35,13 +35,47 @@ def actionoutout_schema_to_mapping(schema: Dict) -> Dict: if property["type"] == "string": mapping[field] = (str, ...) elif property["type"] == "array" and property["items"]["type"] == "string": - mapping[field] = (List[str], ...) + mapping[field] = (list[str], ...) elif property["type"] == "array" and property["items"]["type"] == "array": - # here only consider the `List[List[str]]` situation - mapping[field] = (List[List[str]], ...) + # here only consider the `list[list[str]]` situation + mapping[field] = (list[list[str]], ...) return mapping +def actionoutput_mapping_to_str(mapping: dict) -> dict: + new_mapping = {} + for key, value in mapping.items(): + new_mapping[key] = str(value) + return new_mapping + + +def actionoutput_str_to_mapping(mapping: dict) -> dict: + new_mapping = {} + for key, value in mapping.items(): + if value == "(, Ellipsis)": + new_mapping[key] = (str, ...) + else: + new_mapping[key] = eval(value) # `"'(list[str], Ellipsis)"` to `(list[str], ...)` + return new_mapping + + +def serialize_general_message(message: Message) -> dict: + """ serialize Message, not to save""" + message_cp = copy.deepcopy(message) + ic = message_cp.instruct_content + if ic: + # model create by pydantic create_model like `pydantic.main.prd`, can't pickle.dump directly + schema = ic.schema() + mapping = actionoutout_schema_to_mapping(schema) + mapping = actionoutput_mapping_to_str(mapping) + + message_cp.instruct_content = {"class": schema["title"], "mapping": mapping, "value": ic.dict()} + cb = message_cp.cause_by + if cb: + message_cp.cause_by = cb.ser_class() + return message_cp.dict() + + def serialize_message(message: Message): message_cp = copy.deepcopy(message) # avoid `instruct_content` value update by reference ic = message_cp.instruct_content @@ -56,6 +90,24 @@ def serialize_message(message: Message): return msg_ser +def deserialize_general_message(message_dict: dict) -> Message: + """ deserialize Message, not to load""" + instruct_content = message_dict.pop("instruct_content") + cause_by = message_dict.pop("cause_by") + + message = Message(**message_dict) + if instruct_content: + ic = instruct_content + mapping = actionoutput_str_to_mapping(ic["mapping"]) + ic_obj = ActionOutput.create_model_class(class_name=ic["class"], mapping=mapping) + ic_new = ic_obj(**ic["value"]) + message.instruct_content = ic_new + if cause_by: + message.cause_by = Action.deser_class(cause_by) + + return message + + def deserialize_message(message_ser: str) -> Message: message = pickle.loads(message_ser) if message.instruct_content: diff --git a/metagpt/utils/utils.py b/metagpt/utils/utils.py new file mode 100644 index 000000000..81ceea884 --- /dev/null +++ b/metagpt/utils/utils.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : + +from typing import Any +import json +from pathlib import Path +import importlib + + +def read_json_file(json_file: str, encoding=None) -> list[Any]: + if not Path(json_file).exists(): + raise FileNotFoundError(f"json_file: {json_file} not exist, return []") + + with open(json_file, "r", encoding=encoding) as fin: + try: + data = json.load(fin) + except Exception as exp: + raise ValueError(f"read json file: {json_file} failed") + return data + + +def write_json_file(json_file: str, data: list, encoding=None): + folder_path = Path(json_file).parent + if not folder_path.exists(): + folder_path.mkdir(parents=True, exist_ok=True) + + with open(json_file, "w", encoding=encoding) as fout: + json.dump(data, fout, ensure_ascii=False, indent=4) + + +def import_class(class_name: str, module_name: str) -> type: + module = importlib.import_module(module_name) + a_class = getattr(module, class_name) + return a_class + + +def import_class_inst(class_name: str, module_name: str, *args, **kwargs) -> object: + a_class = import_class(class_name, module_name) + class_inst = a_class(*args, **kwargs) + return class_inst diff --git a/startup.py b/startup.py index e9fbf94d3..9f753d553 100644 --- a/startup.py +++ b/startup.py @@ -4,6 +4,7 @@ import asyncio import fire +from metagpt.const import SERDES_PATH from metagpt.roles import ( Architect, Engineer, @@ -21,26 +22,32 @@ async def startup( code_review: bool = False, run_tests: bool = False, implement: bool = True, + recover_path: bool = False, ): """Run a startup. Be a boss.""" company = Team() - company.hire( - [ - ProductManager(), - Architect(), - ProjectManager(), - ] - ) + if not recover_path: + company.hire( + [ + ProductManager(), + Architect(), + ProjectManager(), + ] + ) - # if implement or code_review - if implement or code_review: - # developing features: implement the idea - company.hire([Engineer(n_borg=5, use_code_review=code_review)]) + # if implement or code_review + if implement or code_review: + # developing features: implement the idea + company.hire([Engineer(n_borg=5, use_code_review=code_review)]) - if run_tests: - # developing features: run tests on the spot and identify bugs - # (bug fixing capability comes soon!) - company.hire([QaEngineer()]) + if run_tests: + # developing features: run tests on the spot and identify bugs + # (bug fixing capability comes soon!) + company.hire([QaEngineer()]) + else: + stg_path = SERDES_PATH.joinpath("team") + company.deserialize(stg_path=stg_path) + idea = company.idea # use original idea company.invest(investment) company.start_project(idea) @@ -54,6 +61,7 @@ def main( code_review: bool = True, run_tests: bool = False, implement: bool = True, + recover_path: str = None, ): """ We are a software startup comprised of AI. By investing in us, @@ -63,9 +71,10 @@ def main( a certain dollar amount to this AI company. :param n_round: :param code_review: Whether to use code review. + :param recover_path: recover the project from existing serialized storage :return: """ - asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) + asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement, recover_path)) if __name__ == "__main__": diff --git a/tests/metagpt/actions/test_action.py b/tests/metagpt/actions/test_action.py index 9775630cc..4468a6f6f 100644 --- a/tests/metagpt/actions/test_action.py +++ b/tests/metagpt/actions/test_action.py @@ -11,3 +11,20 @@ from metagpt.actions import Action, WritePRD, WriteTest def test_action_repr(): actions = [Action(), WriteTest(), WritePRD()] assert "WriteTest" in str(actions) + + +def test_action_serdes(): + action_info = WriteTest.ser_class() + assert action_info["action_class"] == "WriteTest" + + action_class = Action.deser_class(action_info) + assert action_class == WriteTest + + +def test_action_class_serdes(): + name = "write test" + action_info = WriteTest(name=name).serialize() + assert action_info["name"] == name + + action = Action.deserialize(action_info) + assert action.name == name diff --git a/tests/metagpt/memory/test_memory.py b/tests/metagpt/memory/test_memory.py new file mode 100644 index 000000000..bda79ded1 --- /dev/null +++ b/tests/metagpt/memory/test_memory.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of memory + +from pathlib import Path + +from metagpt.schema import Message +from metagpt.memory.memory import Memory +from metagpt.actions.action_output import ActionOutput +from metagpt.actions.design_api import WriteDesign +from metagpt.actions.add_requirement import BossRequirement + +serdes_path = Path(__file__).absolute().parent.joinpath("../../data/serdes_storage") + + +def test_memory_serdes(): + msg1 = Message(role="User", + content="write a 2048 game", + cause_by=BossRequirement) + + out_mapping = {"field1": (list[str], ...)} + out_data = {"field1": ["field1 value1", "field1 value2"]} + ic_obj = ActionOutput.create_model_class("system_design", out_mapping) + msg2 = Message(role="Architect", + instruct_content=ic_obj(**out_data), + content="system design content", + cause_by=WriteDesign) + + memory = Memory() + memory.add_batch([msg1, msg2]) + + stg_path = serdes_path.joinpath("team/environment") + memory.serialize(stg_path) + assert stg_path.joinpath("memory.json").exists() + + new_memory = Memory.deserialize(stg_path) + assert new_memory.count() == 2 + new_msg2 = new_memory.get(1)[0] + assert new_msg2.instruct_content.field1 == ["field1 value1", "field1 value2"] + assert new_msg2.cause_by == WriteDesign + + stg_path.joinpath("memory.json").unlink() diff --git a/tests/metagpt/roles/test_role.py b/tests/metagpt/roles/test_role.py new file mode 100644 index 000000000..a19ad9cb5 --- /dev/null +++ b/tests/metagpt/roles/test_role.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of Role + +from pathlib import Path +import shutil +import pytest + +from metagpt.roles.role import Role, RoleReactMode +from metagpt.actions.action import Action +from metagpt.schema import Message +from metagpt.actions.add_requirement import BossRequirement +from metagpt.roles.product_manager import ProductManager + +serdes_path = Path(__file__).absolute().parent.joinpath("../../data/serdes_storage") + + +def test_role_serdes(): + stg_path_prefix = serdes_path.joinpath("team/environment/roles/") + shutil.rmtree(serdes_path.joinpath("team"), ignore_errors=True) + + pm = ProductManager() + role_tag = f"{pm.__class__.__name__}_{pm.name}" + stg_path = stg_path_prefix.joinpath(role_tag) + pm.serialize(stg_path) + assert stg_path.joinpath("actions/actions_info.json").exists() + + new_pm = Role.deserialize(stg_path) + assert new_pm.name == pm.name + assert len(new_pm.get_memories(1)) == 0 + + +class ActionOK(Action): + + async def run(self, messages: list["Message"]): + return "ok" + + +class ActionRaise(Action): + + async def run(self, messages: list["Message"]): + raise RuntimeError("parse error") + + +class RoleA(Role): + + def __init__(self, + name: str = "RoleA", + profile: str = "Role A", + goal: str = "", + constraints: str = ""): + super(RoleA, self).__init__(name=name, profile=profile, goal=goal, constraints=constraints) + self._init_actions([ActionOK, ActionRaise]) + self._watch([BossRequirement]) + self._rc.react_mode = RoleReactMode.BY_ORDER + + async def run(self, message: "Message" = None, stg_path: str = None): + try: + await super(RoleA, self).run(message) + except Exception as exp: + print("exp ", exp) + self.serialize(stg_path) + + +@pytest.mark.asyncio +async def test_role_serdes_interrupt(): + role_a = RoleA() + shutil.rmtree(serdes_path.joinpath("team"), ignore_errors=True) + + stg_path = serdes_path.joinpath(f"team/environment/roles/{role_a.__class__.__name__}_{role_a.name}") + await role_a.run( + message=Message(content="demo", cause_by=BossRequirement), + stg_path=stg_path + ) + assert role_a._rc.memory.count() == 2 + + assert stg_path.joinpath("actions/todo.json").exists() + + new_role_a: Role = Role.deserialize(stg_path) + assert new_role_a._rc.state == 1 + await role_a.run( + message=Message(content="demo", cause_by=BossRequirement), + stg_path=stg_path + ) + diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index a0f1f6257..3cc2d8a7a 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -7,13 +7,18 @@ """ import pytest +from pathlib import Path +import shutil from metagpt.actions import BossRequirement from metagpt.environment import Environment from metagpt.logs import logger -from metagpt.manager import Manager from metagpt.roles import Architect, ProductManager, Role from metagpt.schema import Message +from tests.metagpt.roles.test_role import RoleA + + +serdes_path = Path(__file__).absolute().parent.joinpath("../data/serdes_storage") @pytest.fixture @@ -36,21 +41,29 @@ def test_get_roles(env: Environment): assert roles == {role1.profile: role1, role2.profile: role2} -def test_set_manager(env: Environment): - manager = Manager() - env.set_manager(manager) - assert env.manager == manager - - @pytest.mark.asyncio async def test_publish_and_process_message(env: Environment): product_manager = ProductManager("Alice", "Product Manager", "做AI Native产品", "资源有限") architect = Architect("Bob", "Architect", "设计一个可用、高效、较低成本的系统,包括数据结构与接口", "资源有限,需要节省成本") env.add_roles([product_manager, architect]) - env.set_manager(Manager()) env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) await env.run(k=2) logger.info(f"{env.history=}") assert len(env.history) > 10 + + +def test_environment_serdes(): + environment = Environment() + role_a = RoleA() + + shutil.rmtree(serdes_path.joinpath("team"), ignore_errors=True) + + stg_path = serdes_path.joinpath("team/environment") + environment.add_role(role_a) + environment.serialize(stg_path) + + new_env: Environment = Environment() + new_env.deserialize(stg_path) + assert len(new_env.roles) == 1 diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 12666e0d3..f515326e8 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -5,7 +5,11 @@ @Author : alexanderwu @File : test_schema.py """ + from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage +from metagpt.actions.action_output import ActionOutput +from metagpt.actions.write_code import WriteCode +from metagpt.utils.serialize import serialize_general_message, deserialize_general_message def test_messages(): @@ -19,3 +23,41 @@ def test_messages(): text = str(msgs) roles = ['user', 'system', 'assistant', 'QA'] assert all([i in text for i in roles]) + + +def test_message_serdes(): + out_mapping = {"field3": (str, ...), "field4": (list[str], ...)} + out_data = {"field3": "field3 value3", "field4": ["field4 value1", "field4 value2"]} + ic_obj = ActionOutput.create_model_class("code", out_mapping) + + message = Message( + content="code", + instruct_content=ic_obj(**out_data), + role="engineer", + cause_by=WriteCode + ) + message_dict = serialize_general_message(message) + assert message_dict["cause_by"] == {"action_class": "WriteCode"} + assert message_dict["instruct_content"] == { + "class": "code", + "mapping": { + "field3": "(, Ellipsis)", + "field4": "(list[str], Ellipsis)" + }, + "value": { + "field3": "field3 value3", + "field4": ["field4 value1", "field4 value2"] + } + } + + new_message = deserialize_general_message(message_dict) + assert new_message.content == message.content + assert new_message.instruct_content == message.instruct_content + assert new_message.cause_by == message.cause_by + assert new_message.instruct_content.field3 == out_data["field3"] + + message = Message(content="code") + message_dict = serialize_general_message(message) + new_message = deserialize_general_message(message_dict) + assert new_message.instruct_content is None + assert new_message.cause_by == "" diff --git a/tests/metagpt/test_team.py b/tests/metagpt/test_team.py new file mode 100644 index 000000000..ab201152c --- /dev/null +++ b/tests/metagpt/test_team.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# @Desc : unittest of team + +from pathlib import Path +import shutil + +from metagpt.team import Team + +from tests.metagpt.roles.test_role import RoleA + +serdes_path = Path(__file__).absolute().parent.joinpath("../data/serdes_storage") + + +def test_team_serdes(): + company = Team() + company.hire([RoleA()]) + + stg_path = serdes_path.joinpath("team") + shutil.rmtree(stg_path, ignore_errors=True) + + company.serialize(stg_path=stg_path) + + new_company = Team() + new_company.deserialize(stg_path) + + assert len(new_company.environment.roles) == 1