From 32af743b36a8e31cf3c4a063a2869ea7da40a6f8 Mon Sep 17 00:00:00 2001 From: better629 Date: Wed, 20 Dec 2023 10:54:49 +0800 Subject: [PATCH] rm metagpt/utils/utils.py --- metagpt/actions/action.py | 4 +- metagpt/environment.py | 3 +- metagpt/memory/memory.py | 3 +- .../postprecess/base_postprecess_plugin.py | 2 +- metagpt/roles/role.py | 3 +- metagpt/schema.py | 3 +- metagpt/team.py | 3 +- metagpt/utils/common.py | 99 ++++++++++++++++- metagpt/utils/repair_llm_raw_output.py | 2 +- metagpt/utils/serialize.py | 2 +- metagpt/utils/utils.py | 102 ------------------ .../serialize_deserialize/test_role.py | 2 +- 12 files changed, 109 insertions(+), 119 deletions(-) delete mode 100644 metagpt/utils/utils.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 8cba18945..9c7fb06e1 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -18,8 +18,8 @@ from metagpt.llm import LLM from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.postprecess.llm_output_postprecess import llm_output_postprecess -from metagpt.utils.common import OutputParser -from metagpt.utils.utils import general_after_log +from metagpt.utils.common import OutputParser, general_after_log + action_subclass_registry = {} diff --git a/metagpt/environment.py b/metagpt/environment.py index 9108cdf06..a3cbe6978 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -20,8 +20,7 @@ from pydantic import BaseModel, Field from metagpt.logs import logger from metagpt.roles.role import Role, role_subclass_registry from metagpt.schema import Message -from metagpt.utils.common import is_subscribed -from metagpt.utils.utils import read_json_file, write_json_file +from metagpt.utils.common import is_subscribed, read_json_file, write_json_file class Environment(BaseModel): diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 198c0970d..66ab5d4e9 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -14,8 +14,7 @@ from typing import Iterable, Set from pydantic import BaseModel, Field from metagpt.schema import Message -from metagpt.utils.common import any_to_str, any_to_str_set -from metagpt.utils.utils import read_json_file, write_json_file +from metagpt.utils.common import any_to_str, any_to_str_set, read_json_file, write_json_file class Memory(BaseModel): diff --git a/metagpt/provider/postprecess/base_postprecess_plugin.py b/metagpt/provider/postprecess/base_postprecess_plugin.py index 0d1cfbb11..afcef2531 100644 --- a/metagpt/provider/postprecess/base_postprecess_plugin.py +++ b/metagpt/provider/postprecess/base_postprecess_plugin.py @@ -44,7 +44,7 @@ class BasePostPrecessPlugin(object): def run_retry_parse_json_text(self, content: str) -> Union[dict, list]: """inherited class can re-implement the function""" - logger.info(f"extracted json CONTENT from output:\n{content}") + # logger.info(f"extracted json CONTENT from output:\n{content}") parsed_data = retry_parse_json_text(output=content) # should use output=content return parsed_data diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 09371ae08..efe3bcbd4 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -37,9 +37,8 @@ from metagpt.memory import Memory from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.human_provider import HumanProvider from metagpt.schema import Message, MessageQueue -from metagpt.utils.common import any_to_str +from metagpt.utils.common import any_to_str, read_json_file, write_json_file, import_class, role_raise_decorator from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output -from metagpt.utils.utils import read_json_file, write_json_file, import_class, role_raise_decorator PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ diff --git a/metagpt/schema.py b/metagpt/schema.py index 0fdc24e02..1c1fdd94d 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -36,10 +36,9 @@ from metagpt.const import ( TASK_FILE_REPO, ) from metagpt.logs import logger -from metagpt.utils.common import any_to_str, any_to_str_set +from metagpt.utils.common import any_to_str, any_to_str_set, import_class from metagpt.utils.serialize import actionoutout_schema_to_mapping, actionoutput_mapping_to_str, \ actionoutput_str_to_mapping -from metagpt.utils.utils import import_class class RawMessage(TypedDict): diff --git a/metagpt/team.py b/metagpt/team.py index 30e3dc618..383f2da36 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -19,8 +19,7 @@ from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.utils.common import NoMoneyException -from metagpt.utils.utils import read_json_file, write_json_file, serialize_decorator +from metagpt.utils.common import NoMoneyException, read_json_file, write_json_file, serialize_decorator class Team(BaseModel): diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index a9bdd6e2d..c909180cc 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -13,12 +13,21 @@ from __future__ import annotations import ast import contextlib +import importlib import inspect +import json import os import platform import re +import traceback +import typing +from pathlib import Path +from typing import Any from typing import List, Tuple, Union +from pydantic.json import pydantic_encoder +from tenacity import _utils + from metagpt.const import MESSAGE_ROUTE_TO_ALL from metagpt.logs import logger @@ -184,7 +193,7 @@ class OutputParser: if start_index != -1 and end_index != -1: # Extract the structure part - structure_text = text[start_index : end_index + 1] + structure_text = text[start_index: end_index + 1] try: # Attempt to convert the text to a Python data type using ast.literal_eval @@ -363,3 +372,91 @@ def is_subscribed(message, tags): if t in message.send_to: return True return False + + +def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: + def log_it(retry_state: "RetryCallState") -> None: + if retry_state.fn is None: + fn_name = "" + else: + fn_name = _utils.get_callback_name(retry_state.fn) + logger.error( + f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " + f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " + f"exp: {retry_state.outcome.exception()}" + ) + + return log_it + + +def read_json_file(json_file: str, encoding=None) -> list[Any]: + if not Path(json_file).exists(): + raise FileNotFoundError(f"json_file: {json_file} not exist, return []") + + with open(json_file, "r", encoding=encoding) as fin: + try: + data = json.load(fin) + except Exception as exp: + raise ValueError(f"read json file: {json_file} failed") + return data + + +def write_json_file(json_file: str, data: list, encoding=None): + folder_path = Path(json_file).parent + if not folder_path.exists(): + folder_path.mkdir(parents=True, exist_ok=True) + + with open(json_file, "w", encoding=encoding) as fout: + json.dump(data, fout, ensure_ascii=False, indent=4, default=pydantic_encoder) + + +def import_class(class_name: str, module_name: str) -> type: + module = importlib.import_module(module_name) + a_class = getattr(module, class_name) + return a_class + + +def import_class_inst(class_name: str, module_name: str, *args, **kwargs) -> object: + a_class = import_class(class_name, module_name) + class_inst = a_class(*args, **kwargs) + return class_inst + + +def format_trackback_info(limit: int = 2): + return traceback.format_exc(limit=limit) + + +def serialize_decorator(func): + async def wrapper(self, *args, **kwargs): + try: + result = await func(self, *args, **kwargs) + self.serialize() # Team.serialize + return result + except KeyboardInterrupt as kbi: + logger.error(f"KeyboardInterrupt occurs, start to serialize the project, exp:\n{format_trackback_info()}") + self.serialize() # Team.serialize + except Exception as exp: + logger.error(f"Exception occurs, start to serialize the project, exp:\n{format_trackback_info()}") + self.serialize() # Team.serialize + + return wrapper + + +def role_raise_decorator(func): + async def wrapper(self, *args, **kwargs): + try: + return await func(self, *args, **kwargs) + except KeyboardInterrupt as kbi: + logger.error(f"KeyboardInterrupt: {kbi} occurs, start to serialize the project") + if self.latest_observed_msg: + self._rc.memory.delete(self.latest_observed_msg) + raise Exception(format_trackback_info(limit=None)) # raise again to make it captured outside + except Exception as exp: + if self.latest_observed_msg: + logger.warning("There is a exception in role's execution, in order to resume, " + "we delete the newest role communication message in the role's memory.") + # remove role newest observed msg to make it observed again + self._rc.memory.delete(self.latest_observed_msg) + raise Exception(format_trackback_info(limit=None)) # raise again to make it captured outside + + return wrapper diff --git a/metagpt/utils/repair_llm_raw_output.py b/metagpt/utils/repair_llm_raw_output.py index 4aafd8e66..67ad4e963 100644 --- a/metagpt/utils/repair_llm_raw_output.py +++ b/metagpt/utils/repair_llm_raw_output.py @@ -253,7 +253,7 @@ def retry_parse_json_text(output: str) -> Union[list, dict]: if CONFIG.repair_llm_output is True, the _aask_v1 and the retry_parse_json_text will loop for {x=3*3} times. it's a two-layer retry cycle """ - logger.debug(f"output to json decode:\n{output}") + # logger.debug(f"output to json decode:\n{output}") # if CONFIG.repair_llm_output is True, it will try to fix output until the retry break parsed_data = CustomDecoder(strict=False).decode(output) diff --git a/metagpt/utils/serialize.py b/metagpt/utils/serialize.py index 93f584057..9a758da34 100644 --- a/metagpt/utils/serialize.py +++ b/metagpt/utils/serialize.py @@ -5,7 +5,7 @@ import copy import pickle -from metagpt.utils.utils import import_class +from metagpt.utils.common import import_class def actionoutout_schema_to_mapping(schema: dict) -> dict: diff --git a/metagpt/utils/utils.py b/metagpt/utils/utils.py deleted file mode 100644 index aa7c039c4..000000000 --- a/metagpt/utils/utils.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# @Desc : - -import typing -from typing import Any -import json -from pathlib import Path -import importlib -from tenacity import _utils -import traceback -from pydantic.json import pydantic_encoder - -from metagpt.logs import logger - - -def general_after_log(logger: "loguru.Logger", sec_format: str = "%0.3f") -> typing.Callable[["RetryCallState"], None]: - def log_it(retry_state: "RetryCallState") -> None: - if retry_state.fn is None: - fn_name = "" - else: - fn_name = _utils.get_callback_name(retry_state.fn) - logger.error( - f"Finished call to '{fn_name}' after {sec_format % retry_state.seconds_since_start}(s), " - f"this was the {_utils.to_ordinal(retry_state.attempt_number)} time calling it. " - f"exp: {retry_state.outcome.exception()}" - ) - - return log_it - - -def read_json_file(json_file: str, encoding=None) -> list[Any]: - if not Path(json_file).exists(): - raise FileNotFoundError(f"json_file: {json_file} not exist, return []") - - with open(json_file, "r", encoding=encoding) as fin: - try: - data = json.load(fin) - except Exception as exp: - raise ValueError(f"read json file: {json_file} failed") - return data - - -def write_json_file(json_file: str, data: list, encoding=None): - folder_path = Path(json_file).parent - if not folder_path.exists(): - folder_path.mkdir(parents=True, exist_ok=True) - - with open(json_file, "w", encoding=encoding) as fout: - json.dump(data, fout, ensure_ascii=False, indent=4, default=pydantic_encoder) - - -def import_class(class_name: str, module_name: str) -> type: - module = importlib.import_module(module_name) - a_class = getattr(module, class_name) - return a_class - - -def import_class_inst(class_name: str, module_name: str, *args, **kwargs) -> object: - a_class = import_class(class_name, module_name) - class_inst = a_class(*args, **kwargs) - return class_inst - - -def format_trackback_info(limit: int = 2): - return traceback.format_exc(limit=limit) - - -def serialize_decorator(func): - async def wrapper(self, *args, **kwargs): - try: - result = await func(self, *args, **kwargs) - self.serialize() # Team.serialize - return result - except KeyboardInterrupt as kbi: - logger.error(f"KeyboardInterrupt occurs, start to serialize the project, exp:\n{format_trackback_info()}") - self.serialize() # Team.serialize - except Exception as exp: - logger.error(f"Exception occurs, start to serialize the project, exp:\n{format_trackback_info()}") - self.serialize() # Team.serialize - - return wrapper - - -def role_raise_decorator(func): - async def wrapper(self, *args, **kwargs): - try: - return await func(self, *args, **kwargs) - except KeyboardInterrupt as kbi: - logger.error(f"KeyboardInterrupt: {kbi} occurs, start to serialize the project") - if self.latest_observed_msg: - self._rc.memory.delete(self.latest_observed_msg) - raise Exception(format_trackback_info(limit=None)) # raise again to make it captured outside - except Exception as exp: - if self.latest_observed_msg: - logger.warning("There is a exception in role's execution, in order to resume, " - "we delete the newest role communication message in the role's memory.") - # remove role newest observed msg to make it observed again - self._rc.memory.delete(self.latest_observed_msg) - raise Exception(format_trackback_info(limit=None)) # raise again to make it captured outside - - return wrapper diff --git a/tests/metagpt/serialize_deserialize/test_role.py b/tests/metagpt/serialize_deserialize/test_role.py index 87cf75caa..88c7f7d8b 100644 --- a/tests/metagpt/serialize_deserialize/test_role.py +++ b/tests/metagpt/serialize_deserialize/test_role.py @@ -15,7 +15,7 @@ from metagpt.roles.engineer import Engineer from metagpt.roles.product_manager import ProductManager from metagpt.roles.role import Role from metagpt.schema import Message -from metagpt.utils.utils import format_trackback_info +from metagpt.utils.common import format_trackback_info from tests.metagpt.serialize_deserialize.test_serdeser_base import RoleA, RoleB, RoleC, serdeser_path