apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-05-15 11:02:36 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #510 from better629/feat_serdes

Browse source 
add mg ser&deser
This commit is contained in:
better629 2023-11-28 10:26:54 +08:00 • committed by GitHub
commit f2aa9f7046
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 748 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

33
metagpt/actions/action.py
View file

@ -17,6 +17,7 @@ from metagpt.llm import LLM
from metagpt.logs import logger
from metagpt.utils.common import OutputParser
from metagpt.utils.custom_decoder import CustomDecoder
from metagpt.utils.utils import import_class
class Action(BaseModel):
@ -42,7 +43,37 @@ class Action(BaseModel):
def __repr__(self):
return self.__str__()
def serialize(self):
return {
"action_class": self.__class__.__name__,
"module_name": self.__module__,
"name": self.name
}
@classmethod
def deserialize(cls, action_dict: dict):
action_class_str = action_dict.pop("action_class")
module_name = action_dict.pop("module_name")
action_class = import_class(action_class_str, module_name)
return action_class(**action_dict)
@classmethod
def ser_class(cls):
""" serialize class type"""
return {
"action_class": cls.__name__,
"module_name": cls.__module__
}
@classmethod
def deser_class(cls, action_dict: dict):
""" deserialize class type """
action_class_str = action_dict.pop("action_class")
module_name = action_dict.pop("module_name")
action_class = import_class(action_class_str, module_name)
return action_class
async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> str:
"""Append default prefix"""
if not system_msgs:

1
metagpt/const.py
View file

@ -42,6 +42,7 @@ TMP = PROJECT_ROOT / "tmp"
RESEARCH_PATH = DATA_PATH / "research"
TUTORIAL_PATH = DATA_PATH / "tutorial_docx"
INVOICE_OCR_TABLE_PATH = DATA_PATH / "invoice_table"
SERDES_PATH = WORKSPACE_ROOT / "storage" # TODO to store `storage` under the individual generated project
SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills"

38
metagpt/environment.py
View file

@ -7,12 +7,14 @@
"""
import asyncio
from typing import Iterable
from pathlib import Path
from pydantic import BaseModel, Field
from metagpt.memory import Memory
from metagpt.roles import Role
from metagpt.schema import Message
from metagpt.utils.utils import read_json_file, write_json_file
class Environment(BaseModel):
@ -28,6 +30,42 @@ class Environment(BaseModel):
class Config:
arbitrary_types_allowed = True
def serialize(self, stg_path: Path):
roles_path = stg_path.joinpath("roles.json")
roles_info = []
for role_key, role in self.roles.items():
roles_info.append({
"role_class": role.__class__.__name__,
"module_name": role.__module__,
"role_name": role.name
})
role.serialize(stg_path=stg_path.joinpath(f"roles/{role.__class__.__name__}_{role.name}"))
write_json_file(roles_path, roles_info)
self.memory.serialize(stg_path)
history_path = stg_path.joinpath("history.json")
write_json_file(history_path, {"content": self.history})
def deserialize(self, stg_path: Path):
""" stg_path: ./storage/team/environment/ """
roles_path = stg_path.joinpath("roles.json")
roles_info = read_json_file(roles_path)
for role_info in roles_info:
role_class = role_info.get("role_class")
role_name = role_info.get("role_name")
role_path = stg_path.joinpath(f"roles/{role_class}_{role_name}")
role = Role.deserialize(role_path)
self.add_role(role)
memory = Memory.deserialize(stg_path)
self.memory = memory
history_path = stg_path.joinpath("history.json")
history = read_json_file(history_path)
self.history = history.get("content")
def add_role(self, role: Role):
"""增加一个在当前环境的角色, 默认为profile/role_profile
Add a role in the current environment

30
metagpt/memory/memory.py
View file

@ -7,9 +7,12 @@
"""
from collections import defaultdict
from typing import Iterable, Type
from pathlib import Path
from metagpt.actions import Action
from metagpt.schema import Message
from metagpt.utils.utils import read_json_file, write_json_file
from metagpt.utils.serialize import serialize_general_message, deserialize_general_message
class Memory:
@ -20,6 +23,33 @@ class Memory:
self.storage: list[Message] = []
self.index: dict[Type[Action], list[Message]] = defaultdict(list)
def serialize(self, stg_path: Path):
""" stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/ """
memory_path = stg_path.joinpath("memory.json")
storage = []
for message in self.storage:
# msg_dict = message.serialize()
msg_dict = serialize_general_message(message)
storage.append(msg_dict)
write_json_file(memory_path, storage)
@classmethod
def deserialize(cls, stg_path: Path) -> "Memory":
""" stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/"""
memory_path = stg_path.joinpath("memory.json")
memory = Memory()
memory_list = read_json_file(memory_path)
for message in memory_list:
# distinguish instruct_content type in message
# msg = Message.deserialize(message)
msg = deserialize_general_message(message)
memory.add(msg)
return memory
def add(self, message: Message):
"""Add a new message to storage, while updating the index"""
if message in self.storage:

246
metagpt/roles/role.py
View file

@ -6,9 +6,11 @@
@File : role.py
"""
import sys
from enum import Enum
import importlib
from __future__ import annotations
import sys
from types import SimpleNamespace
from typing import (
Dict,
@ -28,6 +30,7 @@ from metagpt.llm import LLM
from metagpt.logs import logger
from metagpt.memory import Memory, LongTermMemory
from metagpt.schema import Message
from metagpt.utils.utils import read_json_file, write_json_file, import_class
PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """
@ -37,12 +40,14 @@ Please note that only the text between the first and second "===" is information
{history}
===
You can now choose one of the following stages to decide the stage you need to go in the next step:
Your previous stage: {previous_state}
Now choose one of the following stages you need to go to in the next step:
{states}
Just answer a number between 0-{n_states}, choose the most suitable stage according to the understanding of the conversation.
Please note that the answer only needs a number, no need to add any other text.
If there is no conversation record, choose 0.
If you think you have completed your goal and don't need to go to any of the stages, return -1.
Do not answer anything else, and do not add any other information in your answer.
"""
@ -56,6 +61,15 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi
{name}: {result}
"""
class RoleReactMode(str, Enum):
REACT = "react"
BY_ORDER = "by_order"
PLAN_AND_ACT = "plan_and_act"
@classmethod
def values(cls):
return [item.value for item in cls]
class RoleSetting(BaseModel):
"""Role Settings"""
@ -81,6 +95,8 @@ class RoleContext(BaseModel):
todo: Action = Field(default=None)
watch: set[Type[Action]] = Field(default_factory=set)
news: list[Type[Message]] = Field(default=[])
react_mode: RoleReactMode = RoleReactMode.REACT # see `Role._set_react_mode` for definitions of the following two attributes
max_react_loop: int = 1
class Config:
arbitrary_types_allowed = True
@ -102,6 +118,7 @@ class RoleContext(BaseModel):
class Role(BaseModel):
"""Role/Agent"""
name: str = ""
profile: str = ""
goal: str = ""
@ -116,8 +133,8 @@ class Role(BaseModel):
_rc: RoleContext = RoleContext()
_private_attributes = {
'_setting': _setting,
'_role_id': _role_id,
"_setting': _setting,
"_role_id': _role_id,
'_states': [],
'_actions': [],
'_actions_type': [] # 用于记录和序列化
@ -144,9 +161,7 @@ class Role(BaseModel):
def _reset(self):
object.__setattr__(self, '_states', [])
object.__setattr__(self, '_actions', [])
@staticmethod
def _process_class(class_str, module_name):
cleaned_string = re.sub(r"[<>']", "", class_str).replace("class ", "")
@ -157,37 +172,167 @@ class Role(BaseModel):
module_file = import_module(file_name, package=package_name)
module = getattr(module_file, module_name)
return module
def serialize(self, stg_path: Path):
role_info_path = stg_path.joinpath("role_info.json")
role_info = {
"role_class": self.__class__.__name__,
"module_name": self.__module__
}
setting = self._setting.dict()
setting.pop("desc")
setting.pop("is_human") # not all inherited roles have this atrr
role_info.update(setting)
write_json_file(role_info_path, role_info)
actions_info_path = stg_path.joinpath("actions/actions_info.json")
actions_info = []
for action in self._actions:
actions_info.append(action.serialize())
write_json_file(actions_info_path, actions_info)
watches_info_path = stg_path.joinpath("watches/watches_info.json")
watches_info = []
for watch in self._rc.watch:
watches_info.append(watch.ser_class())
write_json_file(watches_info_path, watches_info)
actions_todo_path = stg_path.joinpath("actions/todo.json")
actions_todo = {
"cur_state": self._rc.state,
"react_mode": self._rc.react_mode.value,
"max_react_loop": self._rc.max_react_loop
}
write_json_file(actions_todo_path, actions_todo)
self._rc.memory.serialize(stg_path)
@classmethod
def deserialize(cls, stg_path: Path) -> "Role":
""" stg_path = ./storage/team/environment/roles/{role_class}_{role_name}"""
role_info_path = stg_path.joinpath("role_info.json")
role_info = read_json_file(role_info_path)
role_class_str = role_info.pop("role_class")
module_name = role_info.pop("module_name")
role_class = import_class(class_name=role_class_str, module_name=module_name)
role = role_class(**role_info) # initiate particular Role
actions_info_path = stg_path.joinpath("actions/actions_info.json")
actions = []
actions_info = read_json_file(actions_info_path)
for action_info in actions_info:
action = Action.deserialize(action_info)
actions.append(action)
watches_info_path = stg_path.joinpath("watches/watches_info.json")
watches = []
watches_info = read_json_file(watches_info_path)
for watch_info in watches_info:
action = Action.deser_class(watch_info)
watches.append(action)
role.init_actions(actions)
role.watch(watches)
actions_todo_path = stg_path.joinpath("actions/todo.json")
# recover self._rc.state
actions_todo = read_json_file(actions_todo_path)
max_react_loop = actions_todo.get("max_react_loop", 1)
cur_state = actions_todo.get("cur_state", -1)
role.set_state(cur_state)
role.set_recovered(True)
react_mode_str = actions_todo.get("react_mode", RoleReactMode.REACT.value)
if react_mode_str not in RoleReactMode.values():
logger.warning(f"ReactMode: {react_mode_str} not in {RoleReactMode.values()}, use react as default")
react_mode_str = RoleReactMode.REACT.value
role.set_react_mode(RoleReactMode(react_mode_str), max_react_loop)
role_memory = Memory.deserialize(stg_path)
role.set_memory(role_memory)
return role
def _reset(self):
self._states = []
self._actions = []
def set_recovered(self, recovered: bool = False):
self._recovered = recovered
def set_memory(self, memory: Memory):
self._rc.memory = memory
def init_actions(self, actions):
self._init_actions(actions)
def _init_actions(self, actions):
self._reset()
for idx, action in enumerate(actions):
if not isinstance(action, Action):
## 默认初始化
i = action()
i = action("", llm=self._llm)
else:
if self._setting.is_human and not isinstance(action.llm, HumanProvider):
logger.warning(f"is_human attribute does not take effect,"
f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances")
i = action
i.set_prefix(self._get_prefix(), self.profile)
self._actions.append(i)
self._states.append(f"{idx}. {action}")
action_title = action.schema()["title"]
self._actions_type.append(action_title)
def set_react_mode(self, react_mode: RoleReactMode, max_react_loop: int = 1):
self._set_react_mode(react_mode, max_react_loop)
def _set_react_mode(self, react_mode: str, max_react_loop: int = 1):
"""Set strategy of the Role reacting to observed Message. Variation lies in how
this Role elects action to perform during the _think stage, especially if it is capable of multiple Actions.
Args:
react_mode (str): Mode for choosing action during the _think stage, can be one of:
"react": standard think-act loop in the ReAct paper, alternating thinking and acting to solve the task, i.e. _think -> _act -> _think -> _act -> ...
Use llm to select actions in _think dynamically;
"by_order": switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...;
"plan_and_act": first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ...
Use llm to come up with the plan dynamically.
Defaults to "react".
max_react_loop (int): Maximum react cycles to execute, used to prevent the agent from reacting forever.
Take effect only when react_mode is react, in which we use llm to choose actions, including termination.
Defaults to 1, i.e. _think -> _act (-> return result and end)
"""
assert react_mode in RoleReactMode.values(), f"react_mode must be one of {RoleReactMode.values()}"
self._rc.react_mode = react_mode
if react_mode == RoleReactMode.REACT:
self._rc.max_react_loop = max_react_loop
def watch(self, actions: Iterable[Type[Action]]):
self._watch(actions)
def _watch(self, actions: Iterable[Type[Action]]):
"""Listen to the corresponding behaviors"""
self._rc.watch.update(actions)
# check RoleContext after adding watch actions
self._rc.check(self._role_id)
def _set_state(self, state):
def set_state(self, state: int):
self._set_state(state)
def _set_state(self, state: int):
"""Update the current state."""
self._rc.state = state
logger.debug(self._actions)
self._rc.todo = self._actions[self._rc.state]
self._rc.todo = self._actions[self._rc.state] if state >= 0 else None
def set_env(self, env: 'Environment'):
"""Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing."""
self._rc.env = env
@property
def name(self):
return self._setting.name
@property
def profile(self):
"""Get the role description (position)"""
@ -205,15 +350,25 @@ class Role(BaseModel):
# If there is only one action, then only this one can be performed
self._set_state(0)
return
if self._recovered and self._rc.state >= 0:
self._set_state(self._rc.state) # action to run from recovered state
self._recovered = False # avoid max_react_loop out of work
return
prompt = self._get_prefix()
prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states),
n_states=len(self._states) - 1)
n_states=len(self._states) - 1, previous_state=self._rc.state)
next_state = await self._llm.aask(prompt)
logger.debug(f"{prompt=}")
if not next_state.isdigit() or int(next_state) not in range(len(self._states)):
logger.warning(f'Invalid answer of state, {next_state=}')
next_state = "0"
self._set_state(int(next_state))
if (not next_state.isdigit() and next_state != "-1") \
or int(next_state) not in range(-1, len(self._states)):
logger.warning(f'Invalid answer of state, {next_state=}, will be set to -1')
next_state = -1
else:
next_state = int(next_state)
if next_state == -1:
logger.info(f"End actions with {next_state=}")
self._set_state(next_state)
async def _act(self) -> Message:
logger.info(f"{self._setting}: ready to {self._rc.todo}")
@ -237,8 +392,7 @@ class Role(BaseModel):
observed = self._rc.env.memory.get_by_actions(self._rc.watch)
self._rc.news = self._rc.memory.find_news(
observed) # find news (previously unseen messages) from observed messages
self._rc.news = self._rc.memory.find_news(observed) # find news (previously unseen messages) from observed messages
for i in env_msgs:
self.recv(i)
@ -256,11 +410,47 @@ class Role(BaseModel):
self._rc.env.publish_message(msg)
async def _react(self) -> Message:
"""Think first, then act"""
await self._think()
logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}")
return await self._act()
"""Think first, then act, until the Role _think it is time to stop and requires no more todo.
This is the standard think-act loop in the ReAct paper, which alternates thinking and acting in task solving, i.e. _think -> _act -> _think -> _act -> ...
Use llm to select actions in _think dynamically
"""
actions_taken = 0
rsp = Message("No actions taken yet") # will be overwritten after Role _act
while actions_taken < self._rc.max_react_loop:
# think
await self._think()
if self._rc.todo is None:
break
# act
logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}")
rsp = await self._act()
actions_taken += 1
return rsp # return output from the last action
async def _act_by_order(self) -> Message:
"""switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ..."""
start_idx = self._rc.state if self._rc.state >= 0 else 0 # action to run from recovered state
for i in range(start_idx, len(self._states)):
self._set_state(i)
rsp = await self._act()
return rsp # return output from the last action
async def _plan_and_act(self) -> Message:
"""first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... Use llm to come up with the plan dynamically."""
# TODO: to be implemented
return Message("")
async def react(self) -> Message:
"""Entry to one of three strategies by which Role reacts to the observed Message"""
if self._rc.react_mode == RoleReactMode.REACT:
rsp = await self._react()
elif self._rc.react_mode == RoleReactMode.BY_ORDER:
rsp = await self._act_by_order()
elif self._rc.react_mode == RoleReactMode.PLAN_AND_ACT:
rsp = await self._plan_and_act()
self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None
return rsp
def recv(self, message: Message) -> None:
"""add message to history."""
# self._history += f"\n{message}"
@ -276,6 +466,10 @@ class Role(BaseModel):
return await self._react()
def get_memories(self, k=0) -> list[Message]:
"""A wrapper to return the most recent k memories of this role, return all when k=0"""
return self._rc.memory.get(k=k)
async def run(self, message=None):
"""Observe, and think and act based on the results of the observation"""
if message:
@ -290,7 +484,7 @@ class Role(BaseModel):
logger.debug(f"{self._setting}: no news. waiting.")
return
rsp = await self._react()
rsp = await self.react()
# Publish the reply to the environment, waiting for the next subscriber to process
self._publish_message(rsp)
return rsp

44
metagpt/schema.py
View file

@ -9,10 +9,14 @@ from __future__ import annotations
from dataclasses import dataclass, field
from typing import Type, TypedDict
import copy
from pydantic import BaseModel
from metagpt.logs import logger
# from metagpt.utils.serialize import actionoutout_schema_to_mapping
# from metagpt.actions.action_output import ActionOutput
# from metagpt.actions.action import Action
class RawMessage(TypedDict):
@ -38,6 +42,46 @@ class Message:
def __repr__(self):
return self.__str__()
# def serialize(self):
# message_cp: Message = copy.deepcopy(self)
# ic = message_cp.instruct_content
# if ic:
# # model create by pydantic create_model like `pydantic.main.prd`, can't pickle.dump directly
# schema = ic.schema()
# mapping = actionoutout_schema_to_mapping(schema)
#
# message_cp.instruct_content = {"class": schema["title"], "mapping": mapping, "value": ic.dict()}
# cb = message_cp.cause_by
# if cb:
# message_cp.cause_by = cb.serialize()
#
# return message_cp.dict()
#
# @classmethod
# def deserialize(cls, message_dict: dict):
# instruct_content = message_dict.get("instruct_content")
# if instruct_content:
# ic = instruct_content
# ic_obj = ActionOutput.create_model_class(class_name=ic["class"], mapping=ic["mapping"])
# ic_new = ic_obj(**ic["value"])
# message_dict.instruct_content = ic_new
# cause_by = message_dict.get("cause_by")
# if cause_by:
# message_dict.cause_by = Action.deserialize(cause_by)
#
# return Message(**message_dict)
def dict(self):
return {
"content": self.content,
"instruct_content": self.instruct_content,
"role": self.role,
"cause_by": self.cause_by,
"sent_from": self.sent_from,
"send_to": self.send_to,
"restricted_to": self.restricted_to
}
def to_dict(self) -> dict:
return {
"role": self.role,

26
metagpt/team.py
View file

@ -5,6 +5,7 @@
@Author : alexanderwu
@File : software_company.py
"""
from pathlib import Path
from pydantic import BaseModel, Field
from metagpt.actions import BossRequirement
@ -14,6 +15,7 @@ from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Message
from metagpt.utils.common import NoMoneyException
from metagpt.utils.utils import read_json_file, write_json_file
class Team(BaseModel):
@ -28,6 +30,30 @@ class Team(BaseModel):
class Config:
arbitrary_types_allowed = True
def serialize(self, stg_path: Path):
team_info_path = stg_path.joinpath("team_info.json")
write_json_file(team_info_path, {
"idea": self.idea,
"investment": self.investment
})
self.environment.serialize(stg_path.joinpath("environment"))
def deserialize(self, stg_path: Path):
""" stg_path = ./storage/team """
# recover team_info
team_info_path = stg_path.joinpath("team_info.json")
if not team_info_path.exists():
logger.error("recover storage not exist, not to recover and continue run the old project.")
team_info = read_json_file(team_info_path)
self.investment = team_info.get("investment", 10.0)
self.idea = team_info.get("idea", "")
# recover environment
environment_path = stg_path.joinpath("environment")
self.environment = Environment()
self.environment.deserialize(stg_path=environment_path)
def hire(self, roles: list[Role]):
"""Hire roles to cooperate"""
self.environment.add_roles(roles)

62
metagpt/utils/serialize.py
View file

@ -4,13 +4,13 @@
import copy
import pickle
from typing import Dict, List
from metagpt.actions.action_output import ActionOutput
from metagpt.schema import Message
from metagpt.actions.action import Action
def actionoutout_schema_to_mapping(schema: Dict) -> Dict:
def actionoutout_schema_to_mapping(schema: dict) -> dict:
"""
directly traverse the `properties` in the first level.
schema structure likes
@ -35,13 +35,47 @@ def actionoutout_schema_to_mapping(schema: Dict) -> Dict:
if property["type"] == "string":
mapping[field] = (str, ...)
elif property["type"] == "array" and property["items"]["type"] == "string":
mapping[field] = (List[str], ...)
mapping[field] = (list[str], ...)
elif property["type"] == "array" and property["items"]["type"] == "array":
# here only consider the `List[List[str]]` situation
mapping[field] = (List[List[str]], ...)
# here only consider the `list[list[str]]` situation
mapping[field] = (list[list[str]], ...)
return mapping
def actionoutput_mapping_to_str(mapping: dict) -> dict:
new_mapping = {}
for key, value in mapping.items():
new_mapping[key] = str(value)
return new_mapping
def actionoutput_str_to_mapping(mapping: dict) -> dict:
new_mapping = {}
for key, value in mapping.items():
if value == "(<class 'str'>, Ellipsis)":
new_mapping[key] = (str, ...)
else:
new_mapping[key] = eval(value) # `"'(list[str], Ellipsis)"` to `(list[str], ...)`
return new_mapping
def serialize_general_message(message: Message) -> dict:
""" serialize Message, not to save"""
message_cp = copy.deepcopy(message)
ic = message_cp.instruct_content
if ic:
# model create by pydantic create_model like `pydantic.main.prd`, can't pickle.dump directly
schema = ic.schema()
mapping = actionoutout_schema_to_mapping(schema)
mapping = actionoutput_mapping_to_str(mapping)
message_cp.instruct_content = {"class": schema["title"], "mapping": mapping, "value": ic.dict()}
cb = message_cp.cause_by
if cb:
message_cp.cause_by = cb.ser_class()
return message_cp.dict()
def serialize_message(message: Message):
message_cp = copy.deepcopy(message) # avoid `instruct_content` value update by reference
ic = message_cp.instruct_content
@ -56,6 +90,24 @@ def serialize_message(message: Message):
return msg_ser
def deserialize_general_message(message_dict: dict) -> Message:
""" deserialize Message, not to load"""
instruct_content = message_dict.pop("instruct_content")
cause_by = message_dict.pop("cause_by")
message = Message(**message_dict)
if instruct_content:
ic = instruct_content
mapping = actionoutput_str_to_mapping(ic["mapping"])
ic_obj = ActionOutput.create_model_class(class_name=ic["class"], mapping=mapping)
ic_new = ic_obj(**ic["value"])
message.instruct_content = ic_new
if cause_by:
message.cause_by = Action.deser_class(cause_by)
return message
def deserialize_message(message_ser: str) -> Message:
message = pickle.loads(message_ser)
if message.instruct_content:

41
metagpt/utils/utils.py Normal file
View file

@ -0,0 +1,41 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc :
from typing import Any
import json
from pathlib import Path
import importlib
def read_json_file(json_file: str, encoding=None) -> list[Any]:
if not Path(json_file).exists():
raise FileNotFoundError(f"json_file: {json_file} not exist, return []")
with open(json_file, "r", encoding=encoding) as fin:
try:
data = json.load(fin)
except Exception as exp:
raise ValueError(f"read json file: {json_file} failed")
return data
def write_json_file(json_file: str, data: list, encoding=None):
folder_path = Path(json_file).parent
if not folder_path.exists():
folder_path.mkdir(parents=True, exist_ok=True)
with open(json_file, "w", encoding=encoding) as fout:
json.dump(data, fout, ensure_ascii=False, indent=4)
def import_class(class_name: str, module_name: str) -> type:
module = importlib.import_module(module_name)
a_class = getattr(module, class_name)
return a_class
def import_class_inst(class_name: str, module_name: str, *args, **kwargs) -> object:
a_class = import_class(class_name, module_name)
class_inst = a_class(*args, **kwargs)
return class_inst

41
startup.py
View file

@ -4,6 +4,7 @@ import asyncio
import fire
from metagpt.const import SERDES_PATH
from metagpt.roles import (
Architect,
Engineer,
@ -21,26 +22,32 @@ async def startup(
code_review: bool = False,
run_tests: bool = False,
implement: bool = True,
recover_path: bool = False,
):
"""Run a startup. Be a boss."""
company = Team()
company.hire(
[
ProductManager(),
Architect(),
ProjectManager(),
]
)
if not recover_path:
company.hire(
[
ProductManager(),
Architect(),
ProjectManager(),
]
)
# if implement or code_review
if implement or code_review:
# developing features: implement the idea
company.hire([Engineer(n_borg=5, use_code_review=code_review)])
# if implement or code_review
if implement or code_review:
# developing features: implement the idea
company.hire([Engineer(n_borg=5, use_code_review=code_review)])
if run_tests:
# developing features: run tests on the spot and identify bugs
# (bug fixing capability comes soon!)
company.hire([QaEngineer()])
if run_tests:
# developing features: run tests on the spot and identify bugs
# (bug fixing capability comes soon!)
company.hire([QaEngineer()])
else:
stg_path = SERDES_PATH.joinpath("team")
company.deserialize(stg_path=stg_path)
idea = company.idea # use original idea
company.invest(investment)
company.start_project(idea)
@ -54,6 +61,7 @@ def main(
code_review: bool = True,
run_tests: bool = False,
implement: bool = True,
recover_path: str = None,
):
"""
We are a software startup comprised of AI. By investing in us,
@ -63,9 +71,10 @@ def main(
a certain dollar amount to this AI company.
:param n_round:
:param code_review: Whether to use code review.
:param recover_path: recover the project from existing serialized storage
:return:
"""
asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement))
asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement, recover_path))
if __name__ == "__main__":

17
tests/metagpt/actions/test_action.py
View file

@ -11,3 +11,20 @@ from metagpt.actions import Action, WritePRD, WriteTest
def test_action_repr():
actions = [Action(), WriteTest(), WritePRD()]
assert "WriteTest" in str(actions)
def test_action_serdes():
action_info = WriteTest.ser_class()
assert action_info["action_class"] == "WriteTest"
action_class = Action.deser_class(action_info)
assert action_class == WriteTest
def test_action_class_serdes():
name = "write test"
action_info = WriteTest(name=name).serialize()
assert action_info["name"] == name
action = Action.deserialize(action_info)
assert action.name == name

42
tests/metagpt/memory/test_memory.py Normal file
View file

@ -0,0 +1,42 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc : unittest of memory
from pathlib import Path
from metagpt.schema import Message
from metagpt.memory.memory import Memory
from metagpt.actions.action_output import ActionOutput
from metagpt.actions.design_api import WriteDesign
from metagpt.actions.add_requirement import BossRequirement
serdes_path = Path(__file__).absolute().parent.joinpath("../../data/serdes_storage")
def test_memory_serdes():
msg1 = Message(role="User",
content="write a 2048 game",
cause_by=BossRequirement)
out_mapping = {"field1": (list[str], ...)}
out_data = {"field1": ["field1 value1", "field1 value2"]}
ic_obj = ActionOutput.create_model_class("system_design", out_mapping)
msg2 = Message(role="Architect",
instruct_content=ic_obj(**out_data),
content="system design content",
cause_by=WriteDesign)
memory = Memory()
memory.add_batch([msg1, msg2])
stg_path = serdes_path.joinpath("team/environment")
memory.serialize(stg_path)
assert stg_path.joinpath("memory.json").exists()
new_memory = Memory.deserialize(stg_path)
assert new_memory.count() == 2
new_msg2 = new_memory.get(1)[0]
assert new_msg2.instruct_content.field1 == ["field1 value1", "field1 value2"]
assert new_msg2.cause_by == WriteDesign
stg_path.joinpath("memory.json").unlink()

85
tests/metagpt/roles/test_role.py Normal file
View file

@ -0,0 +1,85 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc : unittest of Role
from pathlib import Path
import shutil
import pytest
from metagpt.roles.role import Role, RoleReactMode
from metagpt.actions.action import Action
from metagpt.schema import Message
from metagpt.actions.add_requirement import BossRequirement
from metagpt.roles.product_manager import ProductManager
serdes_path = Path(__file__).absolute().parent.joinpath("../../data/serdes_storage")
def test_role_serdes():
stg_path_prefix = serdes_path.joinpath("team/environment/roles/")
shutil.rmtree(serdes_path.joinpath("team"), ignore_errors=True)
pm = ProductManager()
role_tag = f"{pm.__class__.__name__}_{pm.name}"
stg_path = stg_path_prefix.joinpath(role_tag)
pm.serialize(stg_path)
assert stg_path.joinpath("actions/actions_info.json").exists()
new_pm = Role.deserialize(stg_path)
assert new_pm.name == pm.name
assert len(new_pm.get_memories(1)) == 0
class ActionOK(Action):
async def run(self, messages: list["Message"]):
return "ok"
class ActionRaise(Action):
async def run(self, messages: list["Message"]):
raise RuntimeError("parse error")
class RoleA(Role):
def __init__(self,
name: str = "RoleA",
profile: str = "Role A",
goal: str = "",
constraints: str = ""):
super(RoleA, self).__init__(name=name, profile=profile, goal=goal, constraints=constraints)
self._init_actions([ActionOK, ActionRaise])
self._watch([BossRequirement])
self._rc.react_mode = RoleReactMode.BY_ORDER
async def run(self, message: "Message" = None, stg_path: str = None):
try:
await super(RoleA, self).run(message)
except Exception as exp:
print("exp ", exp)
self.serialize(stg_path)
@pytest.mark.asyncio
async def test_role_serdes_interrupt():
role_a = RoleA()
shutil.rmtree(serdes_path.joinpath("team"), ignore_errors=True)
stg_path = serdes_path.joinpath(f"team/environment/roles/{role_a.__class__.__name__}_{role_a.name}")
await role_a.run(
message=Message(content="demo", cause_by=BossRequirement),
stg_path=stg_path
)
assert role_a._rc.memory.count() == 2
assert stg_path.joinpath("actions/todo.json").exists()
new_role_a: Role = Role.deserialize(stg_path)
assert new_role_a._rc.state == 1
await role_a.run(
message=Message(content="demo", cause_by=BossRequirement),
stg_path=stg_path
)

29
tests/metagpt/test_environment.py
View file

@ -7,13 +7,18 @@
"""
import pytest
from pathlib import Path
import shutil
from metagpt.actions import BossRequirement
from metagpt.environment import Environment
from metagpt.logs import logger
from metagpt.manager import Manager
from metagpt.roles import Architect, ProductManager, Role
from metagpt.schema import Message
from tests.metagpt.roles.test_role import RoleA
serdes_path = Path(__file__).absolute().parent.joinpath("../data/serdes_storage")
@pytest.fixture
@ -36,21 +41,29 @@ def test_get_roles(env: Environment):
assert roles == {role1.profile: role1, role2.profile: role2}
def test_set_manager(env: Environment):
manager = Manager()
env.set_manager(manager)
assert env.manager == manager
@pytest.mark.asyncio
async def test_publish_and_process_message(env: Environment):
product_manager = ProductManager("Alice", "Product Manager", "做AI Native产品", "资源有限")
architect = Architect("Bob", "Architect", "设计一个可用、高效、较低成本的系统，包括数据结构与接口", "资源有限，需要节省成本")
env.add_roles([product_manager, architect])
env.set_manager(Manager())
env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement))
await env.run(k=2)
logger.info(f"{env.history=}")
assert len(env.history) > 10
def test_environment_serdes():
environment = Environment()
role_a = RoleA()
shutil.rmtree(serdes_path.joinpath("team"), ignore_errors=True)
stg_path = serdes_path.joinpath("team/environment")
environment.add_role(role_a)
environment.serialize(stg_path)
new_env: Environment = Environment()
new_env.deserialize(stg_path)
assert len(new_env.roles) == 1

42
tests/metagpt/test_schema.py
View file

@ -5,7 +5,11 @@
@Author : alexanderwu
@File : test_schema.py
"""
from metagpt.schema import AIMessage, Message, SystemMessage, UserMessage
from metagpt.actions.action_output import ActionOutput
from metagpt.actions.write_code import WriteCode
from metagpt.utils.serialize import serialize_general_message, deserialize_general_message
def test_messages():
@ -19,3 +23,41 @@ def test_messages():
text = str(msgs)
roles = ['user', 'system', 'assistant', 'QA']
assert all([i in text for i in roles])
def test_message_serdes():
out_mapping = {"field3": (str, ...), "field4": (list[str], ...)}
out_data = {"field3": "field3 value3", "field4": ["field4 value1", "field4 value2"]}
ic_obj = ActionOutput.create_model_class("code", out_mapping)
message = Message(
content="code",
instruct_content=ic_obj(**out_data),
role="engineer",
cause_by=WriteCode
)
message_dict = serialize_general_message(message)
assert message_dict["cause_by"] == {"action_class": "WriteCode"}
assert message_dict["instruct_content"] == {
"class": "code",
"mapping": {
"field3": "(<class 'str'>, Ellipsis)",
"field4": "(list[str], Ellipsis)"
},
"value": {
"field3": "field3 value3",
"field4": ["field4 value1", "field4 value2"]
}
}
new_message = deserialize_general_message(message_dict)
assert new_message.content == message.content
assert new_message.instruct_content == message.instruct_content
assert new_message.cause_by == message.cause_by
assert new_message.instruct_content.field3 == out_data["field3"]
message = Message(content="code")
message_dict = serialize_general_message(message)
new_message = deserialize_general_message(message_dict)
assert new_message.instruct_content is None
assert new_message.cause_by == ""

27
tests/metagpt/test_team.py Normal file
View file

@ -0,0 +1,27 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc : unittest of team
from pathlib import Path
import shutil
from metagpt.team import Team
from tests.metagpt.roles.test_role import RoleA
serdes_path = Path(__file__).absolute().parent.joinpath("../data/serdes_storage")
def test_team_serdes():
company = Team()
company.hire([RoleA()])
stg_path = serdes_path.joinpath("team")
shutil.rmtree(stg_path, ignore_errors=True)
company.serialize(stg_path=stg_path)
new_company = Team()
new_company.deserialize(stg_path)
assert len(new_company.environment.roles) == 1