diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index c8c901eb0..f854f509d 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -10,7 +10,7 @@ from __future__ import annotations from typing import Any, Optional, Union -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from metagpt.actions.action_node import ActionNode from metagpt.llm import LLM @@ -26,19 +26,18 @@ action_subclass_registry = {} class Action(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + name: str = "" llm: BaseGPTAPI = Field(default_factory=LLM, exclude=True) context: Union[dict, CodingContext, CodeSummarizeContext, TestingContext, RunCodeContext, str, None] = "" - prefix = "" # aask*时会加上prefix，作为system_message - desc = "" # for skill manager + prefix: str = "" # aask*时会加上prefix，作为system_message + desc: str = "" # for skill manager node: ActionNode = Field(default=None, exclude=True) # builtin variables builtin_class_name: str = "" - class Config: - arbitrary_types_allowed = True - def __init_with_instruction(self, instruction: str): """Initialize action with instruction""" self.node = ActionNode(key=self.name, expected_type=str, instruction=instruction, example="", schema="raw") @@ -58,8 +57,8 @@ class Action(BaseModel): super().__init_subclass__(**kwargs) action_subclass_registry[cls.__name__] = cls - def dict(self, *args, **kwargs) -> "DictStrAny": - obj_dict = super().dict(*args, **kwargs) + def dict(self, *args, **kwargs) -> dict[str, Any]: + obj_dict = super().model_dump(*args, **kwargs) if "llm" in obj_dict: obj_dict.pop("llm") return obj_dict diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 63f46ad45..0a4e0f123 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -11,7 +11,7 @@ NOTE: You should use typing.List instead of list to do type annotation. Because import json from typing import Any, Dict, List, Optional, Tuple, Type -from pydantic import BaseModel, create_model, root_validator, validator +from pydantic import BaseModel, create_model, field_validator, model_validator from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.config import CONFIG @@ -136,13 +136,15 @@ class ActionNode: """基于pydantic v1的模型动态生成，用来检验结果类型正确性""" new_class = create_model(class_name, **mapping) - @validator("*", allow_reuse=True) + @field_validator("*", mode="before") + @classmethod def check_name(v, field): if field.name not in mapping.keys(): raise ValueError(f"Unrecognized block: {field.name}") return v - @root_validator(pre=True, allow_reuse=True) + @model_validator(mode="before") + @classmethod def check_missing_fields(values): required_fields = set(mapping.keys()) missing_fields = required_fields - set(values.keys()) @@ -269,7 +271,9 @@ class ActionNode: output_class = self.create_model_class(output_class_name, output_data_mapping) if schema == "json": - parsed_data = llm_output_postprecess(output=content, schema=output_class.schema(), req_key=f"[/{TAG}]") + parsed_data = llm_output_postprecess( + output=content, schema=output_class.model_json_schema(), req_key=f"[/{TAG}]" + ) else: # using markdown parser parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) @@ -278,7 +282,7 @@ class ActionNode: return content, instruct_content def get(self, key): - return self.instruct_content.dict()[key] + return self.instruct_content.model_dump()[key] def set_recursive(self, name, value): setattr(self, name, value) @@ -337,7 +341,7 @@ class ActionNode: tmp = {} for _, i in self.children.items(): child = await i.simple_fill(schema=schema, mode=mode, timeout=timeout) - tmp.update(child.instruct_content.dict()) + tmp.update(child.instruct_content.model_dump()) cls = self.create_children_class() self.instruct_content = cls(**tmp) return self diff --git a/metagpt/actions/rebuild_class_view.py b/metagpt/actions/rebuild_class_view.py index 2a6a6a6d9..66bc2c7ab 100644 --- a/metagpt/actions/rebuild_class_view.py +++ b/metagpt/actions/rebuild_class_view.py @@ -50,7 +50,7 @@ class RebuildClassView(Action): # try: # node = await REBUILD_CLASS_VIEW_NODE.fill(context=f"```{code_type}\n{src_code}\n```", llm=self.llm, to=format) - # class_view = node.instruct_content.dict()["Class View"] + # class_view = node.instruct_content.model_dump()["Class View"] # except Exception as e: # class_view = RepoParser.rebuild_class_view(src_code, code_type) # await graph_db.insert(subject=concat_namespace(filename, class_name), predicate=GraphKeyword.HAS_CLASS_VIEW, object_=class_view) diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 9fd392a5c..2b7fe2fdc 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -8,7 +8,7 @@ from typing import Any, Optional import pydantic -from pydantic import Field, root_validator +from pydantic import Field, model_validator from metagpt.actions import Action from metagpt.config import CONFIG, Config @@ -114,10 +114,10 @@ class SearchAndSummarize(Action): engine: Optional[SearchEngineType] = CONFIG.search_engine search_func: Optional[Any] = None search_engine: SearchEngine = None + result: str = "" - result = "" - - @root_validator + @model_validator(mode="before") + @classmethod def validate_engine_and_run_func(cls, values): engine = values.get("engine") search_func = values.get("search_func") diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 47e02b699..0cbb547f6 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -187,7 +187,7 @@ class WritePRD(Action): if not CONFIG.project_name: if isinstance(prd, (ActionOutput, ActionNode)): - ws_name = prd.instruct_content.dict()["Project Name"] + ws_name = prd.instruct_content.model_dump()["Project Name"] else: ws_name = CodeParser.parse_str(block="Project Name", text=prd) CONFIG.project_name = ws_name diff --git a/metagpt/document.py b/metagpt/document.py index 0af3a915c..022e5d6f1 100644 --- a/metagpt/document.py +++ b/metagpt/document.py @@ -17,7 +17,7 @@ from langchain.document_loaders import ( UnstructuredWordDocumentLoader, ) from langchain.text_splitter import CharacterTextSplitter -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from tqdm import tqdm from metagpt.config import CONFIG @@ -117,13 +117,12 @@ class IndexableDocument(Document): Advanced document handling: For vector databases or search engines. """ + model_config = ConfigDict(arbitrary_types_allowed=True) + data: Union[pd.DataFrame, list] content_col: Optional[str] = Field(default="") meta_col: Optional[str] = Field(default="") - class Config: - arbitrary_types_allowed = True - @classmethod def from_path(cls, data_path: Path, content_col="content", meta_col="metadata"): if not data_path.exists(): diff --git a/metagpt/environment.py b/metagpt/environment.py index 0ee85f707..06d9a1b4a 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -15,7 +15,7 @@ import asyncio from pathlib import Path from typing import Iterable, Set -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from metagpt.config import CONFIG from metagpt.logs import logger @@ -29,14 +29,13 @@ class Environment(BaseModel): Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles """ + model_config = ConfigDict(arbitrary_types_allowed=True) + desc: str = Field(default="") # 环境描述 roles: dict[str, Role] = Field(default_factory=dict) members: dict[Role, Set] = Field(default_factory=dict) history: str = "" # For debug - class Config: - arbitrary_types_allowed = True - def __init__(self, **kwargs): roles = [] for role_key, role in kwargs.get("roles", {}).items(): diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 1497b8910..8da6ed84a 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -7,7 +7,7 @@ from typing import Optional -from pydantic import Field +from pydantic import ConfigDict, Field from metagpt.logs import logger from metagpt.memory import Memory @@ -22,13 +22,12 @@ class LongTermMemory(Memory): - update memory when it changed """ + model_config = ConfigDict(arbitrary_types_allowed=True) + memory_storage: MemoryStorage = Field(default_factory=MemoryStorage) rc: Optional["RoleContext"] = None msg_from_recover: bool = False - class Config: - arbitrary_types_allowed = True - def recover_memory(self, role_id: str, rc: "RoleContext"): messages = self.memory_storage.recover_memory(role_id) self.rc = rc diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index bd03786ad..93f1774dc 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -41,7 +41,7 @@ class Memory(BaseModel): def serialize(self, stg_path: Path): """stg_path = ./storage/team/environment/ or ./storage/team/environment/roles/{role_class}_{role_name}/""" memory_path = stg_path.joinpath("memory.json") - storage = self.dict() + storage = self.model_dump() write_json_file(memory_path, storage) @classmethod diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 3e5f268f8..a51fbb020 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -26,7 +26,7 @@ from enum import Enum from pathlib import Path from typing import Any, Iterable, Set, Type -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr from metagpt.actions import Action, ActionOutput from metagpt.actions.action import action_subclass_registry @@ -108,9 +108,7 @@ class RoleContext(BaseModel): RoleReactMode.REACT ) # see `Role._set_react_mode` for definitions of the following two attributes max_react_loop: int = 1 - - class Config: - arbitrary_types_allowed = True + model_config = ConfigDict(arbitrary_types_allowed=True) def check(self, role_id: str): # if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: @@ -134,6 +132,8 @@ role_subclass_registry = {} class Role(BaseModel): """Role/Agent""" + model_config = ConfigDict(arbitrary_types_allowed=True, exclude=["_llm"]) + name: str = "" profile: str = "" goal: str = "" @@ -141,11 +141,11 @@ class Role(BaseModel): desc: str = "" is_human: bool = False - _llm: BaseGPTAPI = Field(default_factory=LLM) # Each role has its own LLM, use different system message - _role_id: str = "" - _states: list[str] = [] - _actions: list[Action] = [] - _rc: RoleContext = Field(default_factory=RoleContext) + _llm: BaseGPTAPI = PrivateAttr(default_factory=LLM) # Each role has its own LLM, use different system message + _role_id: str = PrivateAttr(default="") + _states: list[str] = PrivateAttr(default=[]) + _actions: list[Action] = PrivateAttr(default=[]) + _rc: RoleContext = PrivateAttr(default_factory=RoleContext) subscription: set[str] = set() # builtin variables @@ -154,20 +154,16 @@ class Role(BaseModel): builtin_class_name: str = "" _private_attributes = { - "_llm": None, - "_role_id": _role_id, - "_states": [], - "_actions": [], - "_rc": RoleContext(), - "_subscription": set(), + # "_llm": None, + # "_role_id": _role_id, + # "_states": [], + # "_actions": [], + # "_rc": RoleContext(), + # "_subscription": set(), } __hash__ = object.__hash__ # support Role as hashable type in `Environment.members` - class Config: - arbitrary_types_allowed = True - exclude = ["_llm"] - def __init__(self, **kwargs: Any): for index in range(len(kwargs.get("_actions", []))): current_action = kwargs["_actions"][index] @@ -179,7 +175,7 @@ class Role(BaseModel): current_action = subclass(**current_action) break kwargs["_actions"][index] = current_action - + RoleContext.model_rebuild() super().__init__(**kwargs) # 关于私有变量的初始化 https://github.com/pydantic/pydantic/issues/655 @@ -187,25 +183,25 @@ class Role(BaseModel): self._private_attributes["_role_id"] = str(self._setting) self.subscription = {any_to_str(self), self.name} if self.name else {any_to_str(self)} - for key in self._private_attributes.keys(): - if key in kwargs: - object.__setattr__(self, key, kwargs[key]) - if key == "_rc": - _rc = RoleContext(**kwargs["_rc"]) - object.__setattr__(self, "_rc", _rc) - else: - if key == "_rc": - # # Warning, if use self._private_attributes["_rc"], - # # self._rc will be a shared object between roles, so init one or reset it inside `_reset` - object.__setattr__(self, key, RoleContext()) - else: - object.__setattr__(self, key, self._private_attributes[key]) + # for key in self._private_attributes.keys(): + # if key in kwargs: + # object.__setattr__(self, key, kwargs[key]) + # if key == "_rc": + # _rc = RoleContext(**kwargs["_rc"]) + # object.__setattr__(self, "_rc", _rc) + # else: + # if key == "_rc": + # # # Warning, if use self._private_attributes["_rc"], + # # # self._rc will be a shared object between roles, so init one or reset it inside `_reset` + # object.__setattr__(self, key, RoleContext()) + # else: + # object.__setattr__(self, key, self._private_attributes[key]) self._llm.system_prompt = self._get_prefix() # deserialize child classes dynamically for inherited `role` object.__setattr__(self, "builtin_class_name", self.__class__.__name__) - self.__fields__["builtin_class_name"].default = self.__class__.__name__ + self.model_fields["builtin_class_name"].default = self.__class__.__name__ if "actions" in kwargs: self._init_actions(kwargs["actions"]) @@ -231,7 +227,7 @@ class Role(BaseModel): else stg_path ) - role_info = self.dict(exclude={"_rc": {"memory": True, "msg_buffer": True}, "_llm": True}) + role_info = self.model_dump(exclude={"_rc": {"memory": True, "msg_buffer": True}, "_llm": True}) role_info.update({"role_class": self.__class__.__name__, "module_name": self.__module__}) role_info_path = stg_path.joinpath("role_info.json") write_json_file(role_info_path, role_info) diff --git a/metagpt/schema.py b/metagpt/schema.py index c60247aa1..2930e1815 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -25,7 +25,7 @@ from json import JSONDecodeError from pathlib import Path from typing import Any, Dict, List, Optional, Set, Type, TypeVar -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr from metagpt.config import CONFIG from metagpt.const import ( @@ -108,7 +108,7 @@ class Message(BaseModel): role: str = "user" # system / user / assistant cause_by: str = "" sent_from: str = "" - send_to: Set = Field(default_factory={MESSAGE_ROUTE_TO_ALL}) + send_to: Set = Field(default={MESSAGE_ROUTE_TO_ALL}) def __init__(self, content: str = "", **kwargs): ic = kwargs.get("instruct_content", None) @@ -142,26 +142,26 @@ class Message(BaseModel): new_val = val super().__setattr__(key, new_val) - def dict(self, *args, **kwargs) -> "DictStrAny": + def dict(self, *args, **kwargs) -> dict[str, Any]: """overwrite the `dict` to dump dynamic pydantic model""" - obj_dict = super(Message, self).dict(*args, **kwargs) + obj_dict = super(Message, self).model_dump(*args, **kwargs) ic = self.instruct_content if ic: # compatible with custom-defined ActionOutput - schema = ic.schema() + schema = ic.model_json_schema() # `Documents` contain definitions if "definitions" not in schema: # TODO refine with nested BaseModel mapping = actionoutout_schema_to_mapping(schema) mapping = actionoutput_mapping_to_str(mapping) - obj_dict["instruct_content"] = {"class": schema["title"], "mapping": mapping, "value": ic.dict()} + obj_dict["instruct_content"] = {"class": schema["title"], "mapping": mapping, "value": ic.model_dump()} return obj_dict def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) if self.instruct_content: - return f"{self.role}: {self.instruct_content.dict()}" + return f"{self.role}: {self.instruct_content.model_dump()}" return f"{self.role}: {self.content}" def __repr__(self): @@ -224,19 +224,18 @@ class AIMessage(Message): class MessageQueue(BaseModel): """Message queue which supports asynchronous updates.""" - _queue: Queue = Field(default_factory=Queue) + model_config = ConfigDict(arbitrary_types_allowed=True) - _private_attributes = {"_queue": Queue()} + _queue: Queue = PrivateAttr(default_factory=Queue) - class Config: - arbitrary_types_allowed = True + # _private_attributes = {"_queue": Queue()} - def __init__(self, **kwargs: Any): - for key in self._private_attributes.keys(): - if key in kwargs: - object.__setattr__(self, key, kwargs[key]) - else: - object.__setattr__(self, key, Queue()) + # def __init__(self, **kwargs: Any): + # for key in self._private_attributes.keys(): + # if key in kwargs: + # object.__setattr__(self, key, kwargs[key]) + # else: + # object.__setattr__(self, key, Queue()) def pop(self) -> Message | None: """Pop one message from the queue.""" @@ -312,28 +311,28 @@ class BaseContext(BaseModel, ABC): class CodingContext(BaseContext): filename: str - design_doc: Optional[Document] - task_doc: Optional[Document] - code_doc: Optional[Document] + design_doc: Optional[Document] = None + task_doc: Optional[Document] = None + code_doc: Optional[Document] = None class TestingContext(BaseContext): filename: str code_doc: Document - test_doc: Optional[Document] + test_doc: Optional[Document] = None class RunCodeContext(BaseContext): mode: str = "script" - code: Optional[str] + code: Optional[str] = None code_filename: str = "" - test_code: Optional[str] + test_code: Optional[str] = None test_filename: str = "" command: List[str] = Field(default_factory=list) working_directory: str = "" additional_python_paths: List[str] = Field(default_factory=list) - output_filename: Optional[str] - output: Optional[str] + output_filename: Optional[str] = None + output: Optional[str] = None class RunCodeResult(BaseContext): diff --git a/metagpt/subscription.py b/metagpt/subscription.py index 607cbdb8d..e2b0916ac 100644 --- a/metagpt/subscription.py +++ b/metagpt/subscription.py @@ -1,7 +1,7 @@ import asyncio from typing import AsyncGenerator, Awaitable, Callable -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from metagpt.logs import logger from metagpt.roles import Role @@ -33,10 +33,9 @@ class SubscriptionRunner(BaseModel): >>> asyncio.run(main()) """ - tasks: dict[Role, asyncio.Task] = Field(default_factory=dict) + model_config = ConfigDict(arbitrary_types_allowed=True) - class Config: - arbitrary_types_allowed = True + tasks: dict[Role, asyncio.Task] = Field(default_factory=dict) async def subscribe( self, diff --git a/metagpt/team.py b/metagpt/team.py index fd9af9045..ab9ccc5f8 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -11,7 +11,7 @@ import warnings from pathlib import Path -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field from metagpt.actions import UserRequirement from metagpt.config import CONFIG @@ -34,6 +34,8 @@ class Team(BaseModel): dedicated to env any multi-agent activity, such as collaboratively writing executable code. """ + model_config = ConfigDict(arbitrary_types_allowed=True) + env: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") @@ -45,14 +47,11 @@ class Team(BaseModel): if "env_desc" in kwargs: self.env.desc = kwargs["env_desc"] - class Config: - arbitrary_types_allowed = True - def serialize(self, stg_path: Path = None): stg_path = SERDESER_PATH.joinpath("team") if stg_path is None else stg_path team_info_path = stg_path.joinpath("team_info.json") - write_json_file(team_info_path, self.dict(exclude={"env": True})) + write_json_file(team_info_path, self.model_dump(exclude={"env": True})) self.env.serialize(stg_path.joinpath("environment")) # save environment alone diff --git a/metagpt/tools/search_engine_googleapi.py b/metagpt/tools/search_engine_googleapi.py index b9faf2ced..97e29d78f 100644 --- a/metagpt/tools/search_engine_googleapi.py +++ b/metagpt/tools/search_engine_googleapi.py @@ -9,7 +9,7 @@ from typing import Optional from urllib.parse import urlparse import httplib2 -from pydantic import BaseModel, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from metagpt.config import CONFIG from metagpt.logs import logger @@ -25,15 +25,13 @@ except ImportError: class GoogleAPIWrapper(BaseModel): - google_api_key: Optional[str] = None - google_cse_id: Optional[str] = None + google_api_key: Optional[str] = Field(default=None, validate_default=True) + google_cse_id: Optional[str] = Field(default=None, validate_default=True) loop: Optional[asyncio.AbstractEventLoop] = None executor: Optional[futures.Executor] = None + model_config = ConfigDict(arbitrary_types_allowed=True) - class Config: - arbitrary_types_allowed = True - - @validator("google_api_key", always=True) + @field_validator("google_api_key", mode="before") @classmethod def check_google_api_key(cls, val: str): val = val or CONFIG.google_api_key @@ -45,7 +43,7 @@ class GoogleAPIWrapper(BaseModel): ) return val - @validator("google_cse_id", always=True) + @field_validator("google_cse_id", mode="before") @classmethod def check_google_cse_id(cls, val: str): val = val or CONFIG.google_cse_id diff --git a/metagpt/tools/search_engine_serpapi.py b/metagpt/tools/search_engine_serpapi.py index 750184198..ecbeac336 100644 --- a/metagpt/tools/search_engine_serpapi.py +++ b/metagpt/tools/search_engine_serpapi.py @@ -8,13 +8,15 @@ from typing import Any, Dict, Optional, Tuple import aiohttp -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from metagpt.config import CONFIG class SerpAPIWrapper(BaseModel): - search_engine: Any #: :meta private: + model_config = ConfigDict(arbitrary_types_allowed=True) + + search_engine: Any = None #: :meta private: params: dict = Field( default={ "engine": "google", @@ -23,13 +25,11 @@ class SerpAPIWrapper(BaseModel): "hl": "en", } ) - serpapi_api_key: Optional[str] = None + # should add `validate_default=True` to check with default value + serpapi_api_key: Optional[str] = Field(default=None, validate_default=True) aiosession: Optional[aiohttp.ClientSession] = None - class Config: - arbitrary_types_allowed = True - - @validator("serpapi_api_key", always=True) + @field_validator("serpapi_api_key", mode="before") @classmethod def check_serpapi_api_key(cls, val: str): val = val or CONFIG.serpapi_api_key diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index 0eec2694b..de0a203ff 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -9,21 +9,19 @@ import json from typing import Any, Dict, Optional, Tuple import aiohttp -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, ConfigDict, Field, field_validator from metagpt.config import CONFIG class SerperWrapper(BaseModel): - search_engine: Any #: :meta private: + search_engine: Any = None #: :meta private: payload: dict = Field(default={"page": 1, "num": 10}) - serper_api_key: Optional[str] = None + serper_api_key: Optional[str] = Field(default=None, validate_default=True) aiosession: Optional[aiohttp.ClientSession] = None + model_config = ConfigDict(arbitrary_types_allowed=True) - class Config: - arbitrary_types_allowed = True - - @validator("serper_api_key", always=True) + @field_validator("serper_api_key", mode="before") @classmethod def check_serper_api_key(cls, val: str): val = val or CONFIG.serper_api_key diff --git a/metagpt/utils/parse_html.py b/metagpt/utils/parse_html.py index f2395026f..65aa3f236 100644 --- a/metagpt/utils/parse_html.py +++ b/metagpt/utils/parse_html.py @@ -5,7 +5,7 @@ from typing import Generator, Optional from urllib.parse import urljoin, urlparse from bs4 import BeautifulSoup -from pydantic import BaseModel +from pydantic import BaseModel, PrivateAttr class WebPage(BaseModel): @@ -13,11 +13,8 @@ class WebPage(BaseModel): html: str url: str - class Config: - underscore_attrs_are_private = True - - _soup: Optional[BeautifulSoup] = None - _title: Optional[str] = None + _soup: Optional[BeautifulSoup] = PrivateAttr(default=None) + _title: Optional[str] = PrivateAttr(default=None) @property def soup(self) -> BeautifulSoup: diff --git a/metagpt/utils/serialize.py b/metagpt/utils/serialize.py index 3939b1306..4b976e387 100644 --- a/metagpt/utils/serialize.py +++ b/metagpt/utils/serialize.py @@ -62,7 +62,7 @@ def serialize_message(message: "Message"): ic = message_cp.instruct_content if ic: # model create by pydantic create_model like `pydantic.main.prd`, can't pickle.dump directly - schema = ic.schema() + schema = ic.model_json_schema() mapping = actionoutout_schema_to_mapping(schema) message_cp.instruct_content = {"class": schema["title"], "mapping": mapping, "value": ic.dict()} diff --git a/requirements.txt b/requirements.txt index 5cb01ab99..b75fc0fa6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ fire==0.4.0 typer # godot==0.1.1 # google_api_python_client==2.93.0 -lancedb==0.1.16 +lancedb==0.4.0 langchain==0.0.352 loguru==0.6.0 meilisearch==0.21.0 @@ -19,7 +19,7 @@ openai==1.6.0 openpyxl beautifulsoup4==4.12.2 pandas==2.0.3 -pydantic==1.10.8 +pydantic==2.5.3 #pygame==2.1.3 #pymilvus==2.2.8 pytest==7.2.2 @@ -33,16 +33,15 @@ tqdm==4.64.0 #unstructured[local-inference] # selenium>4 # webdriver_manager<3.9 -anthropic==0.3.6 +anthropic==0.8.1 typing-inspect==0.8.0 -aiofiles -typing_extensions==4.7.0 +typing_extensions==4.9.0 libcst==1.0.1 -qdrant-client==1.4.0 +qdrant-client==1.7.0 pytest-mock==3.11.1 # open-interpreter==0.1.7; python_version>"3.9" ta==0.10.2 -semantic-kernel==0.4.0.dev0 +semantic-kernel==0.4.3.dev0 wrapt==1.15.0 #aiohttp_jinja2 #azure-cognitiveservices-speech~=1.31.0 diff --git a/tests/metagpt/serialize_deserialize/test_architect_deserialize.py b/tests/metagpt/serialize_deserialize/test_architect_deserialize.py index b92eba8a1..60d048998 100644 --- a/tests/metagpt/serialize_deserialize/test_architect_deserialize.py +++ b/tests/metagpt/serialize_deserialize/test_architect_deserialize.py @@ -10,7 +10,7 @@ from metagpt.roles.architect import Architect def test_architect_serialize(): role = Architect() - ser_role_dict = role.dict(by_alias=True) + ser_role_dict = role.model_dump(by_alias=True) assert "name" in ser_role_dict assert "_states" in ser_role_dict assert "_actions" in ser_role_dict @@ -19,7 +19,7 @@ def test_architect_serialize(): @pytest.mark.asyncio async def test_architect_deserialize(): role = Architect() - ser_role_dict = role.dict(by_alias=True) + ser_role_dict = role.model_dump(by_alias=True) new_role = Architect(**ser_role_dict) # new_role = Architect.deserialize(ser_role_dict) assert new_role.name == "Bob" diff --git a/tests/metagpt/serialize_deserialize/test_environment.py b/tests/metagpt/serialize_deserialize/test_environment.py index 096c1dd68..d3a668b76 100644 --- a/tests/metagpt/serialize_deserialize/test_environment.py +++ b/tests/metagpt/serialize_deserialize/test_environment.py @@ -20,14 +20,14 @@ from tests.metagpt.serialize_deserialize.test_serdeser_base import ( def test_env_serialize(): env = Environment() - ser_env_dict = env.dict() + ser_env_dict = env.model_dump() assert "roles" in ser_env_dict def test_env_deserialize(): env = Environment() env.publish_message(message=Message(content="test env serialize")) - ser_env_dict = env.dict() + ser_env_dict = env.model_dump() new_env = Environment(**ser_env_dict) assert len(new_env.roles) == 0 assert len(new_env.history) == 25 @@ -47,7 +47,7 @@ def test_environment_serdeser(): environment.add_role(role_c) environment.publish_message(message) - ser_data = environment.dict() + ser_data = environment.model_dump() assert ser_data["roles"]["Role C"]["name"] == "RoleC" new_env: Environment = Environment(**ser_data) @@ -64,7 +64,7 @@ def test_environment_serdeser_v2(): pm = ProjectManager() environment.add_role(pm) - ser_data = environment.dict() + ser_data = environment.model_dump() new_env: Environment = Environment(**ser_data) role = new_env.get_role(pm.profile) diff --git a/tests/metagpt/serialize_deserialize/test_memory.py b/tests/metagpt/serialize_deserialize/test_memory.py index 5a40f5c3b..2a66434e1 100644 --- a/tests/metagpt/serialize_deserialize/test_memory.py +++ b/tests/metagpt/serialize_deserialize/test_memory.py @@ -25,7 +25,7 @@ def test_memory_serdeser(): memory = Memory() memory.add_batch([msg1, msg2]) - ser_data = memory.dict() + ser_data = memory.model_dump() new_memory = Memory(**ser_data) assert new_memory.count() == 2 diff --git a/tests/metagpt/serialize_deserialize/test_product_manager.py b/tests/metagpt/serialize_deserialize/test_product_manager.py index b65e329d1..5cf714688 100644 --- a/tests/metagpt/serialize_deserialize/test_product_manager.py +++ b/tests/metagpt/serialize_deserialize/test_product_manager.py @@ -12,7 +12,7 @@ from metagpt.schema import Message @pytest.mark.asyncio async def test_product_manager_deserialize(): role = ProductManager() - ser_role_dict = role.dict(by_alias=True) + ser_role_dict = role.model_dump(by_alias=True) new_role = ProductManager(**ser_role_dict) assert new_role.name == "Alice" diff --git a/tests/metagpt/serialize_deserialize/test_project_manager.py b/tests/metagpt/serialize_deserialize/test_project_manager.py index e52e3f247..9d4880e86 100644 --- a/tests/metagpt/serialize_deserialize/test_project_manager.py +++ b/tests/metagpt/serialize_deserialize/test_project_manager.py @@ -11,7 +11,7 @@ from metagpt.roles.project_manager import ProjectManager def test_project_manager_serialize(): role = ProjectManager() - ser_role_dict = role.dict(by_alias=True) + ser_role_dict = role.model_dump(by_alias=True) assert "name" in ser_role_dict assert "_states" in ser_role_dict assert "_actions" in ser_role_dict @@ -20,7 +20,7 @@ def test_project_manager_serialize(): @pytest.mark.asyncio async def test_project_manager_deserialize(): role = ProjectManager() - ser_role_dict = role.dict(by_alias=True) + ser_role_dict = role.model_dump(by_alias=True) new_role = ProjectManager(**ser_role_dict) assert new_role.name == "Eve" diff --git a/tests/metagpt/serialize_deserialize/test_role.py b/tests/metagpt/serialize_deserialize/test_role.py index 343f01ace..c9f82136c 100644 --- a/tests/metagpt/serialize_deserialize/test_role.py +++ b/tests/metagpt/serialize_deserialize/test_role.py @@ -34,7 +34,7 @@ def test_roles(): def test_role_serialize(): role = Role() - ser_role_dict = role.dict(by_alias=True) + ser_role_dict = role.model_dump(by_alias=True) assert "name" in ser_role_dict assert "_states" in ser_role_dict assert "_actions" in ser_role_dict @@ -42,7 +42,7 @@ def test_role_serialize(): def test_engineer_serialize(): role = Engineer() - ser_role_dict = role.dict(by_alias=True) + ser_role_dict = role.model_dump(by_alias=True) assert "name" in ser_role_dict assert "_states" in ser_role_dict assert "_actions" in ser_role_dict @@ -51,7 +51,7 @@ def test_engineer_serialize(): @pytest.mark.asyncio async def test_engineer_deserialize(): role = Engineer(use_code_review=True) - ser_role_dict = role.dict(by_alias=True) + ser_role_dict = role.model_dump(by_alias=True) new_role = Engineer(**ser_role_dict) assert new_role.name == "Alex" diff --git a/tests/metagpt/serialize_deserialize/test_schema.py b/tests/metagpt/serialize_deserialize/test_schema.py index 0358265a9..dc55abf09 100644 --- a/tests/metagpt/serialize_deserialize/test_schema.py +++ b/tests/metagpt/serialize_deserialize/test_schema.py @@ -31,7 +31,7 @@ def test_message_without_postprocess(): out_data = {"field1": ["field1 value1", "field1 value2"]} ic_obj = ActionNode.create_model_class("code", out_mapping) message = MockMessage(content="code", instruct_content=ic_obj(**out_data)) - ser_data = message.dict() + ser_data = message.model_dump() assert ser_data["instruct_content"] == {"field1": ["field1 value1", "field1 value2"]} new_message = MockMessage(**ser_data) diff --git a/tests/metagpt/serialize_deserialize/test_team.py b/tests/metagpt/serialize_deserialize/test_team.py index dc41fa4ed..fd7e2e582 100644 --- a/tests/metagpt/serialize_deserialize/test_team.py +++ b/tests/metagpt/serialize_deserialize/test_team.py @@ -33,7 +33,7 @@ def test_team_deserialize(): ] ) assert len(company.env.get_roles()) == 3 - ser_company = company.dict() + ser_company = company.model_dump() new_company = Team(**ser_company) assert len(new_company.env.get_roles()) == 3 @@ -71,7 +71,7 @@ async def test_team_recover(): company.run_project(idea) await company.run(n_round=4) - ser_data = company.dict() + ser_data = company.model_dump() new_company = Team(**ser_data) new_role_c = new_company.env.get_role(role_c.profile) diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index 0ab34437d..f1919d610 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -38,7 +38,7 @@ class TestGetProjectRoot: def test_any_to_str(self): class Input(BaseModel): - x: Any + x: Any = None want: str inputs = [ @@ -56,7 +56,7 @@ class TestGetProjectRoot: def test_any_to_str_set(self): class Input(BaseModel): - x: Any + x: Any = None want: Set inputs = [ diff --git a/tests/metagpt/utils/test_dependency_file.py b/tests/metagpt/utils/test_dependency_file.py index ae4d40ea5..0ff5e97b0 100644 --- a/tests/metagpt/utils/test_dependency_file.py +++ b/tests/metagpt/utils/test_dependency_file.py @@ -21,8 +21,8 @@ from metagpt.utils.dependency_file import DependencyFile async def test_dependency_file(): class Input(BaseModel): x: Union[Path, str] - deps: Optional[Set[Union[Path, str]]] - key: Optional[Union[Path, str]] + deps: Optional[Set[Union[Path, str]]] = None + key: Optional[Union[Path, str]] = None want: Set[str] inputs = [